text:
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;

namespace DigitalPlatform.HTTP
{
    public static class HttpProcessor
    {
        // this formats the HTTP response...
        public static void WriteResponse(Stream stream, HttpResponse response)
        {
            if (response.Content == null)
            {
                response.Content = new byte[] { };
            }

            // default to text/html content type
            if (!response.Headers.ContainsKey("Content-Type"))
            {
                response.Headers["Content-Type"] = "text/html";
            }

            response.Headers["Content-Length"] = response.Content.Length.ToString();

            Write(stream, string.Format("HTTP/1.0 {0} {1}\r\n", response.StatusCode, response.ReasonPhrase));
            Write(stream, string.Join("\r\n", response.Headers.Select(x => string.Format("{0}: {1}", x.Key, x.Value))));
            Write(stream, "\r\n\r\n");

            stream.Write(response.Content, 0, response.Content.Length);
        }

        public static async Task WriteResponseAsync(Stream stream,
            HttpResponse response,
            CancellationToken token)
        {
            if (response.Content == null)
            {
                response.Content = new byte[] { };
            }

            // default to text/html content type
            if (!response.Headers.ContainsKey("Content-Type"))
            {
                response.Headers["Content-Type"] = "text/html";
            }

            response.Headers["Content-Length"] = response.Content.Length.ToString();

            Write(stream, string.Format("HTTP/1.0 {0} {1}\r\n", response.StatusCode, response.ReasonPhrase));
            Write(stream, string.Join("\r\n", response.Headers.Select(x => string.Format("{0}: {1}", x.Key, x.Value))));
            Write(stream, "\r\n\r\n");

            await stream.WriteAsync(response.Content, 0, response.Content.Length, token).ConfigureAwait(false);
        }

        static int MAX_HEADER_LINES = 100;
        static int MAX_ENTITY_BYTES = 1024 * 1024;   // 1M bytes

        public delegate bool Delegate_verifyHeaders(Dictionary<string, string> headers);

        public static async Task<HttpRequest> GetIncomingRequest(Stream inputStream,
            List<byte> cache,
            Delegate_verifyHeaders proc_verify,
            CancellationToken token)
        {
            // Read Request Line
            string request = await ReadLineAsync(inputStream, cache, token).ConfigureAwait(false);
            if (string.IsNullOrEmpty(request))
                return null;    // 表示前端已经切断通讯
#if NO
            string[] tokens = request.Split(' ');
            if (tokens.Length != 3)
            {
                throw new Exception("invalid http request line");
            }
#endif
            List<string> tokens = MessageUtility.SplitFirstLine(request);
            if (tokens == null)
                throw new Exception("1 invalid http request line '" + request + "'");

            string method = tokens[0].ToUpper();
            string url = tokens[1];
            string protocolVersion = tokens[2];

            //Read Headers
            Dictionary<string, string> headers = new Dictionary<string, string>();
            string line;
            while ((line = await ReadLineAsync(inputStream, cache, token).ConfigureAwait(false)) != null)
            {
                if (line.Equals(""))
                {
                    break;
                }

                int separator = line.IndexOf(':');
                if (separator == -1)
                {
                    throw new Exception("1 invalid http header line: " + line);
                }
                string name = line.Substring(0, separator);
                int pos = separator + 1;
                while ((pos < line.Length) && (line[pos] == ' '))
                {
                    pos++;
                }

                string value = line.Substring(pos, line.Length - pos);
                headers.Add(name, value);
                if (headers.Count > MAX_HEADER_LINES)
                    throw new Exception("headers 行数超过配额");
            }

            if (proc_verify != null)
            {
                if (proc_verify(headers) == false)
                    return null;
            }

            byte[] raw_content = null;
            if (headers.ContainsKey("Content-Length"))
            {
                int totalBytes = Convert.ToInt32(headers["Content-Length"]);

                if (totalBytes >= MAX_ENTITY_BYTES)
                    throw new Exception("Content-Length " + totalBytes + " 超过配额");

                int bytesLeft = totalBytes;
                byte[] bytes = new byte[totalBytes];

                while (bytesLeft > 0)
                {
                    byte[] buffer = new byte[bytesLeft > 1024 ? 1024 : bytesLeft];

                    int nRet = 0;
                    if (cache.Count > 0)
                    {
                        nRet = Math.Min(buffer.Length, cache.Count);
                        Array.Copy(cache.ToArray(), buffer, nRet);
                        cache.RemoveRange(0, nRet);
                    }

                    int n = 0;
                    if (nRet < buffer.Length)
                    {
                        // int n = inputStream.Read(buffer, 0, buffer.Length);
                        n = await inputStream.ReadAsync(buffer, nRet, buffer.Length - nRet, token).ConfigureAwait(false);
                    }

                    buffer.CopyTo(bytes, totalBytes - bytesLeft);

                    bytesLeft -= nRet + n;
                }

                raw_content = bytes;
            }

            return new HttpRequest()
            {
                Method = method,
                Url = url,
                Version = protocolVersion,
                Headers = headers,
                Content = raw_content
            };
        }

        // 将一个请求发送给指定的地址,并获得响应
        public static HttpResponse WebCall(HttpRequest request,
            string target_url)
        {
            // http://localhost/dp2library/xe/basic
            UriBuilder builder = new UriBuilder(target_url);

            string strHostName = builder.Host;
            int nPort = builder.Port;
            if (nPort == -1)
                nPort = 80;

            TcpClient client = new TcpClient(AddressFamily.InterNetwork);
            client.Connect(strHostName, nPort);

            request.Url = builder.Path;
            WriteRequest(client.GetStream(), request);

            return GetResponse(client.GetStream());
        }

        // TODO: 可以根据 http header 中的 _timeout 实现超时机制
        // 将一个请求发送给指定的地址,并获得响应
        public static async Task<HttpResponse> WebCallAsync(HttpRequest request,
            string target_url,
            CancellationToken token)
        {
            // http://localhost/dp2library/xe/basic
            UriBuilder builder = new UriBuilder(target_url);

            string strHostName = builder.Host;
            int nPort = builder.Port;
            if (nPort == -1)
                nPort = 80;

            TcpClient client = new TcpClient(AddressFamily.InterNetwork);
            await client.ConnectAsync(strHostName, nPort).ConfigureAwait(false);

            request.Url = builder.Path;
            await WriteRequestAsync(client.GetStream(), request, token).ConfigureAwait(false);

            return await GetResponseAsync(client.GetStream(), token).ConfigureAwait(false);
        }

        // 向 dp2library 发出请求
        private static void WriteRequest(Stream stream,
            HttpRequest request)
        {
            if (request.Content == null)
            {
                request.Content = new byte[] { };
            }

#if NO
            // default to text/html content type
            if (!response.Headers.ContainsKey("Content-Type"))
            {
                response.Headers["Content-Type"] = "text/html";
            }
#endif

            request.Headers["Content-Length"] = request.Content.Length.ToString();

            Write(stream, string.Format("{0} {1} HTTP/1.0\r\n", request.Method, request.Url));
            Write(stream, string.Join("\r\n", request.Headers.Select(x => string.Format("{0}: {1}", x.Key, x.Value))));
            Write(stream, "\r\n\r\n");

            stream.Write(request.Content, 0, request.Content.Length);
        }

        // 向 dp2library 发出请求
        private static async Task WriteRequestAsync(Stream stream,
            HttpRequest request,
            CancellationToken token)
        {
            if (request.Content == null)
            {
                request.Content = new byte[] { };
            }

            request.Headers["Content-Length"] = request.Content.Length.ToString();

            await WriteAsync(stream,
                string.Format("{0} {1} HTTP/1.0\r\n", request.Method, request.Url),
                token).ConfigureAwait(false);
            await WriteAsync(stream,
                string.Join("\r\n", request.Headers.Select(x => string.Format("{0}: {1}", x.Key, x.Value))),
                token).ConfigureAwait(false);
            await WriteAsync(stream,
                "\r\n\r\n",
                token).ConfigureAwait(false);

            await stream.WriteAsync(request.Content, 0, request.Content.Length, token).ConfigureAwait(false);
        }

        // 从 dp2library 获得响应
        private static HttpResponse GetResponse(Stream inputStream)
        {
            // Read Response Line
            string first_line = Readline(inputStream);

#if NO
            // HTTP/1.1 404 Not Found
            string[] tokens = request.Split(' ');
            if (tokens.Length != 3)
            {
                throw new Exception("1 invalid http response line '"+request+"'");
            }
#endif
            List<string> tokens = MessageUtility.SplitFirstLine(first_line);
            if (tokens == null)
                throw new Exception("1 invalid http response line '" + first_line + "'");

            string version = tokens[0].ToUpper();
            string status_code = tokens[1];
            string reason_phrase = tokens[2];

            //Read Headers
            Dictionary<string, string> headers = new Dictionary<string, string>();
            string line;
            while ((line = Readline(inputStream)) != null)
            {
                if (line.Equals(""))
                {
                    break;
                }

                int separator = line.IndexOf(':');
                if (separator == -1)
                {
                    throw new Exception("2 invalid http header line: " + line);
                }
                string name = line.Substring(0, separator);
                int pos = separator + 1;
                while ((pos < line.Length) && (line[pos] == ' '))
                {
                    pos++;
                }

                string value = line.Substring(pos, line.Length - pos);
                headers.Add(name, value);
            }

            byte[] raw_content = null;
            if (headers.ContainsKey("Content-Length"))
            {
                int totalBytes = Convert.ToInt32(headers["Content-Length"]);
                int bytesLeft = totalBytes;
                byte[] bytes = new byte[totalBytes];

                while (bytesLeft > 0)
                {
                    byte[] buffer = new byte[bytesLeft > 1024 ? 1024 : bytesLeft];
                    int n = inputStream.Read(buffer, 0, buffer.Length);
                    buffer.CopyTo(bytes, totalBytes - bytesLeft);

                    bytesLeft -= n;
                }

                raw_content = bytes;
            }

            return new HttpResponse()
            {
                StatusCode = status_code,
                ReasonPhrase = reason_phrase,
                Headers = headers,
                Content = raw_content
            };
        }

        // 从 dp2library 获得响应
        private static async Task<HttpResponse> GetResponseAsync(Stream inputStream,
            CancellationToken token)
        {
            // Read Response Line
            string first_line = await ReadLineAsync0(inputStream, token).ConfigureAwait(false);

#if NO
            // HTTP/1.1 404 Not Found
            string[] tokens = request.Split(' ');
            if (tokens.Length != 3)
            {
                throw new Exception("1 invalid http response line '"+request+"'");
            }
#endif
            List<string> tokens = MessageUtility.SplitFirstLine(first_line);
            if (tokens == null)
                throw new Exception("1 invalid http response line '" + first_line + "'");

            string version = tokens[0].ToUpper();
            string status_code = tokens[1];
            string reason_phrase = tokens[2];

            //Read Headers
            Dictionary<string, string> headers = new Dictionary<string, string>();
            string line;
            while ((line = await ReadLineAsync0(inputStream, token).ConfigureAwait(false)) != null)
            {
                if (line.Equals(""))
                {
                    break;
                }

                int separator = line.IndexOf(':');
                if (separator == -1)
                {
                    throw new Exception("2 invalid http header line: " + line);
                }
                string name = line.Substring(0, separator);
                int pos = separator + 1;
                while ((pos < line.Length) && (line[pos] == ' '))
                {
                    pos++;
                }

                string value = line.Substring(pos, line.Length - pos);
                headers.Add(name, value);
            }

            byte[] raw_content = null;
            if (headers.ContainsKey("Content-Length"))
            {
                int totalBytes = Convert.ToInt32(headers["Content-Length"]);
                // TODO 观察配额
                int bytesLeft = totalBytes;
                byte[] bytes = new byte[totalBytes];

                while (bytesLeft > 0)
                {
                    byte[] buffer = new byte[bytesLeft > 1024 ? 1024 : bytesLeft];
                    int n = await inputStream.ReadAsync(buffer, 0, buffer.Length, token).ConfigureAwait(false);
                    buffer.CopyTo(bytes, totalBytes - bytesLeft);

                    bytesLeft -= n;
                }

                raw_content = bytes;
            }

            return new HttpResponse()
            {
                StatusCode = status_code,
                ReasonPhrase = reason_phrase,
                Headers = headers,
                Content = raw_content
            };
        }

        static int MAX_LINE_CHARS = 500;

        private static string Readline(Stream stream)
        {
            int next_char;
            string data = "";
            while (true)
            {
                next_char = stream.ReadByte();
                if (next_char == '\n') { break; }
                if (next_char == '\r') { continue; }
                if (next_char == -1) { Thread.Sleep(1); continue; };
                data += Convert.ToChar(next_char);

                if (data.Length >= MAX_LINE_CHARS)
                    throw new Exception("头字段行长度超过配额");
            }
            return data;
        }

        static async Task<string> ReadLineAsync0(Stream stream,
            CancellationToken token)
        {
            byte[] buffer = new byte[1];

            int next_char;
            string data = "";
            while (true)
            {
                // next_char = stream.ReadByte();

                int nRet = await stream.ReadAsync(buffer, 0, 1, token).ConfigureAwait(false);
                if (nRet < 1)
                    break;
                next_char = buffer[0];

                if (next_char == '\n') { break; }
                if (next_char == '\r') { continue; }

                data += Convert.ToChar(next_char);

                if (data.Length >= MAX_LINE_CHARS)
                    throw new Exception("头字段行长度超过配额");
            }
            return data;
        }

        /*
2016/11/20 10:41:15 ip:127.0.0.1 TestHandleClient() 异常: System.ObjectDisposedException:无法访问已释放的对象。
对象名:"System.Net.Sockets.NetworkStream"。
   在 System.Net.Sockets.NetworkStream.EndRead(IAsyncResult asyncResult)
   在 System.Threading.Tasks.TaskFactory`1.FromAsyncTrimPromise`1.Complete(TInstance thisRef, Func`3 endMethod, IAsyncResult asyncResult, Boolean requiresSynchronization)
--- 引发异常的上一位置中堆栈跟踪的末尾 ---
   在 System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)
   在 System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
   在 System.Runtime.CompilerServices.TaskAwaiter`1.GetResult()
   在 DigitalPlatform.HTTP.HttpProcessor.<ReadLineAsync>d__43.MoveNext() 位置 c:\chord\chord\DigitalPlatform.HTTP\HttpProcessor.cs:行号 471
--- 引发异常的上一位置中堆栈跟踪的末尾 ---
   在 System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)
   在 System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
   在 System.Runtime.CompilerServices.TaskAwaiter`1.GetResult()
   在 DigitalPlatform.HTTP.HttpProcessor.<GetIncomingRequest>d__8.MoveNext() 位置 c:\chord\chord\DigitalPlatform.HTTP\HttpProcessor.cs:行号 70
--- 引发异常的上一位置中堆栈跟踪的末尾 ---
   在 System.Runtime.CompilerServices.TaskAwaiter.ThrowForNonSuccess(Task task)
   在 System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task task)
   在 System.Runtime.CompilerServices.TaskAwaiter`1.GetResult()
   在 dp2Router.HttpServer.<TestHandleClient>d__7.MoveNext() 位置 c:\chord\chord\dp2Router\HttpServer.cs:行号 117
         * */
        static async Task<string> ReadLineAsync(Stream stream,
    List<byte> cache,
    CancellationToken token)
        {
            byte[] buffer = new byte[4096];

            StringBuilder result = new StringBuilder();
            while (true)
            {
                if (cache.Count == 0)
                {
                    int nRet = await stream.ReadAsync(buffer, 0, buffer.Length, token).ConfigureAwait(false);
                    if (nRet <= 0)
                        break;

                    // 追加到 cache 中
                    {
                        int i = 0;
                        foreach (byte c in buffer)
                        {
                            if (i >= nRet)
                                break;
                            cache.Add(c);
                            i++;
                        }

                    }
                }

                int remove_count = 0;
                try
                {
                    int i = 0;
                    foreach (byte c in cache)
                    {
                        if (c == '\n')
                        {
                            remove_count++;
                            goto END1;
                        }
                        if (c == '\r')
                        {
                            remove_count++;
                            continue;
                        }

                        result.Append(Convert.ToChar(c));
                        remove_count++;
                    }
                }
                finally
                {
                    cache.RemoveRange(0, remove_count);
                }

                if (result.Length >= MAX_LINE_CHARS)
                    throw new Exception("头字段行长度超过配额");
            }
        END1:
            return result.ToString();
        }

        private static void Write(Stream stream, string text)
        {
            byte[] bytes = Encoding.UTF8.GetBytes(text);
            stream.Write(bytes, 0, bytes.Length);
        }

        private static async Task WriteAsync(Stream stream, string text, CancellationToken token)
        {
            byte[] bytes = Encoding.UTF8.GetBytes(text);
            await stream.WriteAsync(bytes, 0, bytes.Length, token).ConfigureAwait(false);
        }
    }
}


text:
package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"golang.org/x/crypto/ssh/terminal"

	"github.com/cheggaaa/pb"
)

type Media struct {
	Id        string `json:"id"`
	MediaType string `json:"media_type"`
	Format    string `json:"format"`
	Status    string `json:"status"`
	Retries   int    `json:"retries"`
}

type APIResponse struct {
	Status       int     `json:"status"`
	Media        []Media `json:"media"`
	Total        int     `json:"total"`
	Limit        int     `json:"limit"`
	Offset       int     `json:"offset"`
	UsingCache   bool    `json:"using_cache"`
	ResponseTime int     `json:"response_time"`
}

var (
	loginUrl    *url.URL
	signinUrl   *url.URL
	apiPageUrl  *url.URL
	apiUrl      *url.URL
	originalUrl *url.URL

	signinValues url.Values

	accessTokenRE *regexp.Regexp
	accessToken   string

	pathPerm os.FileMode = 0770
	filePerm os.FileMode = 0770

	mediaPath string = "picturelife"
	indexPath string = "pl_index.json"

	// Flags
	retryFlag  bool = false // Retry failed images and videos?
	helpFlag   bool = false // Print help text
	statusFlag bool = false // Print current status
	resetFlag  bool = false // Reset everything and start over
)

func init() {
	var err error

	flag.BoolVar(&retryFlag, "retry", retryFlag, "Retry failed images and videos?")
	flag.BoolVar(&helpFlag, "help", helpFlag, "Print help text")
	flag.BoolVar(&statusFlag, "status", statusFlag, "Print current status")
	flag.BoolVar(&resetFlag, "reset", resetFlag, "Reset everything and start over")

	loginUrl, err = url.Parse("http://picturelife.com/login")
	if err != nil {
		panic("Unable to parse login URL")
	}

	// Login posts to this
	signinUrl, err = url.Parse("http://picturelife.com/signin")
	if err != nil {
		panic("Unable to parse sign in URL")
	}

	apiPageUrl, err = url.Parse("http://picturelife.com/api")
	if err != nil {
		panic("Unable to parse API Page URL")
	}

	originalUrl, err = url.Parse("http://picturelife.com/d/original/")
	if err != nil {
		panic("Unable to parse API Page URL")
	}

	accessTokenRE = regexp.MustCompile("<script>\\s*pl\\.access_token\\s*=\\s*'([^']+)';\\s*pl\\.api_url\\s*=\\s*'([^']+)'\\s*</script>")

	err = os.MkdirAll(mediaPath, pathPerm)
	if err != nil {
		panic(err)
	}
}

func main() {
	flag.Parse()

	if helpFlag {
		printHelp()
		return
	}

	if statusFlag {
		printStatus()
		return
	}

	if resetFlag {
		fmt.Print("Resetting...")

		if _, err := os.Stat(indexPath); err == nil {
			os.Remove(indexPath)
		}

		if _, err := os.Stat(mediaPath); err == nil {
			os.RemoveAll(mediaPath)
		}
		fmt.Println(" Done!")
		return
	}

	// Instantiate the crawler
	client := NewCrawler()

	// Ask for email and password
	signinValues := getCredentials()

	res := client.GetOrDie(loginUrl.String())
	res.Body.Close()

	res = client.PostFormOrDie(signinUrl.String(), signinValues)
	body, err := ioutil.ReadAll(res.Body)
	res.Body.Close()

	if strings.Contains(string(body), "Login error! Please check your email and password.") {
		fmt.Println("Login error! Please check your email and password.")
		return
	}

	res = client.GetOrDie(apiPageUrl.String())
	body, err = ioutil.ReadAll(res.Body)
	res.Body.Close()

	fmt.Print("Trying to extract Access Token and API URL...")
	parts := accessTokenRE.FindStringSubmatch(string(body))
	if len(parts) != 3 {
		fmt.Println("\nUnable to extract Access Token and API URL.")
		fmt.Println("This is the source code received:")
		fmt.Println(string(body))
		return
	}
	fmt.Println(" Done!")

	accessToken = parts[1]
	apiUrl, err = url.Parse(parts[2])
	if err != nil {
		fmt.Println("Unable to parse API Page URL")
		return
	}

	// So far, so good... Now extract the index json, if it hasn't already been done

	// If the JSON index file does not exist, we'll fetch it from the API and create it
	if _, err := os.Stat(indexPath); os.IsNotExist(err) {
		fmt.Println("\nTrying to extract index of all files...")

		var progress *pb.ProgressBar
		var allMedia []Media

		indexUrl := apiUrl.String() + "/media/index"

		offset := 0
		limit := 500
		total := -1

		formValues := url.Values{
			"taken_at_after":      {"0"},
			"include_hidden":      {"true"},
			"show_invisible":      {"true"},
			"warm_thumbs":         {"false"},
			"include_names":       {"false"},
			"include_comments":    {"false"},
			"include_signature":   {"false"},
			"include_access_info": {"false"},
			"include_likes":       {"false"},
			"offset":              {strconv.Itoa(offset)},
			"limit":               {strconv.Itoa(limit)},
			"access_token":        {accessToken},
		}

		for total == -1 || offset < total {
			formValues.Set("offset", strconv.Itoa(offset))

			res := client.PostFormOrDie(indexUrl, formValues)
			body, err = ioutil.ReadAll(res.Body)
			res.Body.Close()

			var apiResponse APIResponse
			err := json.Unmarshal(body, &apiResponse)
			if err != nil {
				fmt.Println("ERROR! Unable to read JSON response from API. Please try again later.")
				os.Exit(0)
			}

			allMedia = append(allMedia, apiResponse.Media...)
			total = apiResponse.Total

			if progress == nil {
				progress = pb.New(total)
				progress.ShowCounters = true
				progress.ShowTimeLeft = true
				progress.Start()
			}

			progress.Set(offset)

			offset += limit
		}

		progress.FinishPrint("Done fetching JSON index")

		fmt.Print("Writing status file...")
		mediaJson, err := json.Marshal(allMedia)

		err = ioutil.WriteFile(indexPath, mediaJson, filePerm)
		fmt.Println(" Done!")

		if err != nil {
			fmt.Println("ERROR! Unable to write JSON index file to disk. Sorry...")
			fmt.Println("Please go to GitHub and open an issue.")
			os.Exit(0)
		}
	}

	if _, err := os.Stat(indexPath); os.IsNotExist(err) {
		fmt.Println("ERROR! Unable to find the JSON index file from disk. Sorry...")
		fmt.Println("Please go to GitHub and open an issue.")
		os.Exit(0)
	}

	src, err := ioutil.ReadFile(indexPath)
	if err != nil {
		fmt.Println("ERROR! Unable to read the JSON index file from disk. Sorry...")
		fmt.Println("Please go to GitHub and open an issue.")
		os.Exit(0)
	}

	var allMedia []Media
	json.Unmarshal(src, &allMedia)

	fmt.Println("\nTrying to extract pictures and videos...")

	progressCount := len(allMedia)
	for i, media := range allMedia {
		filePath := mediaPath + "/" + getMediaFilename(&media)

		if _, err := os.Stat(filePath); os.IsNotExist(err) {
			allMedia[i].Status = ""
		} else if media.Status == "started" {
			allMedia[i].Status = ""
		} else if media.Status == "done" {
			progressCount--
		} else if !retryFlag && media.Status == "failed" {
			progressCount--
		}
	}

	mediaJson, _ := json.Marshal(allMedia)
	err = ioutil.WriteFile(indexPath, mediaJson, filePerm)

	ch := make(chan bool, 10)
	mediaLock := sync.Mutex{}

	progress := pb.New(progressCount)
	progress.ShowCounters = true
	progress.ShowTimeLeft = true
	progress.Start()

	fails := 0
	success := 0
	for i, media := range allMedia {
		if allMedia[i].Status == "done" {
			success += 1
			continue
		}

		if !retryFlag && allMedia[i].Status == "failed" {
			fails += 1
			continue
		}

		if i > 0 && i%10 == 0 {
			mediaLock.Lock()
			mediaJson, _ := json.Marshal(allMedia)
			err = ioutil.WriteFile(indexPath, mediaJson, filePerm)
			mediaLock.Unlock()

			if err != nil {
				fmt.Println("ERROR! Unable to write update JSON index file to disk. Sorry...")
				fmt.Println("Please go to GitHub and open an issue.")
				os.Exit(0)
			}
		}

		ch <- true

		go func(ch chan bool, index int, media Media) {
			mediaLock.Lock()
			allMedia[index].Status = "started"
			mediaLock.Unlock()
			fetchMedia(&client, &media)
			mediaLock.Lock()
			allMedia[index] = media
			if media.Status == "done" {
				success += 1
			} else {
				fails += 1
			}
			progress.Increment()
			mediaLock.Unlock()
			<-ch
		}(ch, i, media)
	}

	// Wait for the last routines to be done
	for len(ch) > 0 {
		time.Sleep(500 * time.Millisecond)
	}

	mediaLock.Lock()
	mediaJson, _ = json.Marshal(allMedia)
	err = ioutil.WriteFile(indexPath, mediaJson, filePerm)
	mediaLock.Unlock()

	if err != nil {
		fmt.Println("ERROR! Unable to write update JSON index file to disk. Sorry...")
		fmt.Println("Please go to GitHub and open an issue.")
		os.Exit(0)
	}

	progress.Finish()

	fmt.Println("Done trying to fetch all pictures and videos.")
	fmt.Println("Result:")
	fmt.Println("\tSuccess:", success)
	fmt.Println("\tFailed: ", fails)
}

func fetchMedia(client *Crawler, media *Media) {
	media.Retries += 1
	media.Status = "started"

	filename := getMediaFilename(media)

	filePath := mediaPath + "/" + filename
	url := originalUrl.String() + media.Id

	out, err := os.Create(filePath)
	if err != nil {
		media.Status = "failed"
		out.Close()
		os.Remove(filePath)
		return
	}

	res, err := client.Client.Get(url)
	if err != nil || res.StatusCode != 200 {
		media.Status = "failed"
		out.Close()
		if res != nil {
			res.Body.Close()
		}
		os.Remove(filePath)
		return
	}

	n, err := io.Copy(out, res.Body)
	if err != nil {
		media.Status = "failed"
		out.Close()
		res.Body.Close()
		os.Remove(filePath)
		return
	}

	if n < 1000 {
		media.Status = "failed"
		out.Close()
		res.Body.Close()
		os.Remove(filePath)

	} else {
		media.Status = "done"
		out.Close()
		res.Body.Close()
	}
}

func getMediaFilename(media *Media) (filename string) {
	extension := strings.ToLower(media.Format)
	extension = strings.Replace(extension, "jpeg", "jpg", 1)
	if strings.Trim(extension, " ") == "" {
		if media.MediaType == "Video" {
			// Assume mov
			extension = "mov"
		}
	}
	filename = media.Id + "." + extension
	return
}

func printHelp() {
	fmt.Println("Currently you can only choose whether or not to retry failed fetches")
	flag.PrintDefaults()
	fmt.Println("")
	fmt.Println("Usage:")
	fmt.Println(`./rescuelife -retry`)
	fmt.Println("")
}

func printStatus() {
	if _, err := os.Stat(indexPath); os.IsNotExist(err) {
		fmt.Println("ERROR! Unable to find the JSON index file from disk. Sorry...")
		return
	}

	src, err := ioutil.ReadFile(indexPath)
	if err != nil {
		fmt.Println("ERROR! Unable to read the JSON index file from disk. Sorry...")
		return
	}

	var allMedia []Media

	json.Unmarshal(src, &allMedia)

	var failed, started, done, waiting int
	total := len(allMedia)
	for _, media := range allMedia {
		switch media.Status {
		case "done":
			done++
		case "started":
			started++
		case "failed":
			failed++
		default:
			waiting++
		}
	}

	fmt.Println("\nStatus for fetching")
	fmt.Println("-----------------------------")
	fmt.Println("Succeeded:", done)
	fmt.Println("Failed:   ", failed)
	fmt.Println("Fetching: ", started)
	fmt.Println("Waiting:  ", waiting)
	fmt.Println("Total:    ", total)
	fmt.Println("")
}

func getCredentials() (signinValues url.Values) {
	fmt.Println("\n---------------------------------------------------------------------------------------------------------------------")
	fmt.Println("Your email and password is needed in order to get a cookie, extract Access Token and to fetch your images and videos.")
	fmt.Println("Nothing will be stored or copied to any other server.")
	fmt.Println("---------------------------------------------------------------------------------------------------------------------\n")

	reader := bufio.NewReader(os.Stdin)

	fmt.Print("Your email: ")
	email, _ := reader.ReadString('\n')
	email = strings.Trim(email, "\n")

	fmt.Print("Your password: ")
	bytePassword, _ := terminal.ReadPassword(0)
	password := strings.Trim(string(bytePassword), "\n")
	fmt.Println("\n")

	if email == "" || password == "" {
		fmt.Println("ERROR! Please provide email and password")
		os.Exit(0)
	}

	signinValues = url.Values{"email": {email}, "password": {password}}

	return
}

redpajama_set_name: RedPajamaGithub


text:
package org.zouzias.spark.lucenerdd.facets

import com.holdenkarau.spark.testing.SharedSparkContext
import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.zouzias.spark.lucenerdd.testing.FavoriteCaseClass
import org.zouzias.spark.lucenerdd.{LuceneRDD, LuceneRDDKryoRegistrator}

import org.scalatest.BeforeAndAfterEach
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest._
import matchers.should._

class FacetedLuceneRDDImplicitsSpec  extends AnyFlatSpec
  with Matchers
  with BeforeAndAfterEach
  with SharedSparkContext {

  var luceneRDD: LuceneRDD[_] = _

  override val conf = LuceneRDDKryoRegistrator.registerKryoClasses(new SparkConf().
    setMaster("local[*]").
    setAppName("test").
    set("spark.ui.enabled", "false").
    set("spark.app.id", appID))

  override def afterEach() {
    luceneRDD.close()
  }

  val elem = Array("fear", "death", "water", "fire", "house")
    .zipWithIndex.map{ case (str, index) =>
    FavoriteCaseClass(str, index, 10L, 12.3F, s"${str}@gmail.com")}

  "FacetedLuceneRDD(case class).count" should "return correct number of elements" in {
    val rdd = sc.parallelize(elem)
    val spark = SparkSession.builder().getOrCreate()
    import spark.implicits._
    val df = rdd.toDF()
    luceneRDD = FacetedLuceneRDD(df)
    luceneRDD.count should equal (elem.size)
  }

  "FacetedLuceneRDD(case class).fields" should "return all fields" in {
    val rdd = sc.parallelize(elem)
    val spark = SparkSession.builder().getOrCreate()
    import spark.implicits._
    val df = rdd.toDF()
    luceneRDD = FacetedLuceneRDD(df)

    luceneRDD.fields().size should equal(5)
    luceneRDD.fields().contains("name") should equal(true)
    luceneRDD.fields().contains("age") should equal(true)
    luceneRDD.fields().contains("myLong") should equal(true)
    luceneRDD.fields().contains("myFloat") should equal(true)
    luceneRDD.fields().contains("email") should equal(true)
  }

  "FacetedLuceneRDD(case class).termQuery" should "correctly search with TermQueries" in {
    val rdd = sc.parallelize(elem)
    val spark = SparkSession.builder().getOrCreate()
    import spark.implicits._
    val df = rdd.toDF()
    luceneRDD = FacetedLuceneRDD(df)

    val results = luceneRDD.termQuery("name", "water")
    results.count() should equal(1)
  }
}


text:
import pathlib

import setuptools

subpackage_name = "ipython"

def read_version():
    root_path = pathlib.Path(__file__).parent
    version_path = root_path / "abjadext" / subpackage_name / "_version.py"
    with version_path.open() as file_pointer:
        file_contents = file_pointer.read()
    local_dict = {}
    exec(file_contents, None, local_dict)
    return local_dict["__version__"]

if __name__ == "__main__":
    setuptools.setup(
        author="Josiah Wolf Oberholtzer",
        author_email="josiah.oberholtzer@gmail.com",
        classifiers=[
            "Development Status :: 3 - Alpha",
            "License :: OSI Approved :: GNU General Public License (GPL)",
            "Programming Language :: Python :: 3.6",
            "Programming Language :: Python :: Implementation :: CPython",
            "Topic :: Artistic Software",
        ],
        include_package_data=True,
        install_requires=[
            "abjad==3.3",
            "black",
            "flake8",
            "isort",
            "mypy",
            "pytest>=5.4.3",
            "pytest-helpers-namespace",
            "jupyter>=1.0.0",
        ],
        license="MIT",
        long_description=pathlib.Path("README.md").read_text(),
        keywords=", ".join(
            [
                "jupyter",
                "lilypond",
                "music composition",
                "music notation",
            ]
        ),
        name="abjad-ext-{}".format(subpackage_name),
        packages=["abjadext"],
        platforms="Any",
        url="http://abjad.github.io",
        version=read_version(),
    )


text:
// Copyright(c) 2005-2006. All Rights Reserved
// By Jean-René Bédard (https://github.com/jrbedard/3d-converter)

#ifndef __STLREADER_H__
#define __STLREADER_H__

#include "reader.h"
#include "stlFile.h"

namespace ZBPlugin
{

class CFile;

// CStlReader: STL file reader
class CStlReader : public CReader
{
public:
	CStlReader(CFile* pFile, const fs::path& fileName);

private:
	CStlReader(const CStlReader&); // Copy constructor
	void operator=(const CStlReader&); // Assignment operator

public:
	bool Read();
	bool ReadDebug();

private:
	CStlFile* m_pStlFile;
};

} // End ZBPlugin namespace

#endif // __STLREADER_H__


text:
"""
Defines intervals for holding components.

:copyright: 2015 Agile Geoscience
:license: Apache 2.0
"""
import operator
import warnings
from functools import total_ordering

try:
    from functools import partialmethod
except:  # Python 2
    from utils import partialmethod

from .component import Component
from .position import Position
from . import utils

class IntervalError(Exception):
    """
    Generic error class.
    """
    pass

@total_ordering
class Interval(object):
    """
    Used to represent a lithologic or stratigraphic interval, or single point,
    such as a sample location.

    Initialize with a top (and optional base) and a description and/or
    an ordered list of components.

    Args:
        top (float): Required top depth. Required.
        base (float): Base depth. Optional.
        description (str): Textual description.
        lexicon (dict): A lexicon. See documentation. Optional unless you only
            provide descriptions, because it's needed to extract components.
        max_component (int): The number of components to extract. Default 1.
        abbreviations (bool): Whether to parse for abbreviations.

    TODO:
        Seems like I should be able to instantiate like this:

            ``Interval({'top': 0, 'components':[Component({'age': 'Neogene'})``s

        I can get around it for now like this:

            ``Interval(**{'top': 0, 'components':[Component({'age': 'Neogene'})``

        Question: should Interval itself cope with only being handed 'top' and
        either fill in down to the next or optionally create a point?
    """
    def __init__(self, top, base=None,
                 description='',
                 lexicon=None,
                 data=None,
                 components=None,
                 max_component=1,
                 abbreviations=False):

        if not isinstance(top, Position):
            top = Position(middle=top)

        if base is not None:
            if not isinstance(base, Position):
                base = Position(middle=base)

        self.top = top
        if base is not None:
            self.base = base
        else:
            self.base = top

        self.description = str(description)

        self.data = data or {}

        if components:
            self.components = list(components)
        else:
            self.components = []

        if self.description and (not self.components):
            if lexicon:
                comps = self._parse_description(self.description,
                                            lexicon,
                                            max_component=max_component,
                                            abbreviations=abbreviations
                                           )
                self.components = comps
            else:
                with warnings.catch_warnings():
                    w = "You must provide a lexicon to generate "
                    w += "components from descriptions."
                    warnings.warn(w)
                self.components = []

    def __setattr__(self, name, value):
        # If we were passed top or base, make sure it's a position.
        if name in ['top', 'base']:
            if not isinstance(value, Position):
                value = Position(middle=value)
        # Must now use the parent's setattr, or we go in circles.
        super(Interval, self).__setattr__(name, value)
        return

    def __str__(self):
        return self.__dict__.__str__()

    def __repr__(self):
        s = str(self)
        return "Interval({0})".format(s)

    def __add__(self, other):
        """
        TODO:
            If adding components, should take account of 'amount', if present.
            Or 'proportion'? ...Could be specified by lexicon??
        """
        if isinstance(other, self.__class__):
            return self.union(other)

        elif isinstance(other, Component):
            top = self.top.z
            base = self.base.z
            d = self.description + ' with ' + other.summary()
            c = self.components + [other]
            data = self._combine_data(other)

            return Interval(top, base, description=d, data=data, components=c)

        else:
            m = "You can only add components or intervals."
            raise IntervalError(m)

    def __eq__(self, other):
        """
        Must supply __eq__ and one other rich comparison for
        the total_ordering function to provide the others.
        """
        if isinstance(other, self.__class__):
            return self.top == other.top

    def __lt__(self, other):
        if isinstance(other, self.__class__):
            if self.order == 'elevation':
                return self.top < other.top
            return self.top > other.top

    def __bool__(self):
        if (not self.components) and (not self.data):
            return False
        else:
            return True

    def _repr_html_(self):
        """
        Jupyter Notebook magic repr function.
        """
        items = ['top', 'primary', 'summary', 'description', 'data', 'base']
        rows = ''
        row = '
{row1}{e}	{v}
'
        style = 'width:2em; background-color:#DDDDDD'
        extra = '	'
        for i, e in enumerate(items):
            row1 = extra.format(style, len(items)) if not i else ''
            v = getattr(self, e)
            v = v._repr_html_() if (v and (e == 'primary')) else v
            v = self.summary() if e == 'summary' else v
            v = utils.dict_repr_html(self.data) if e == 'data' else v
            v = v.z if e in ['top', 'base'] else v
            rows += row.format(row1=row1, e=e, v=v)

        html = '
{}
'.format(rows)
        return html

    @property
    def primary(self):
        """
        Convenience function returning the first component.

        Returns:
            Component. The first one in the list of components.
        """
        if self.components:
            return self.components[0]
        else:
            return None

    @property
    def middle(self):
        """
        Returns the middle of the interval.

        Returns:
            Float: The middle.
        """
        return (self.base.z + self.top.z) / 2

    @property
    def thickness(self):
        """
        Returns the thickness of the interval.

        Returns:
            Float: The thickness.
        """
        return abs(self.base.z - self.top.z)

    @property
    def min_thickness(self):
        """
        Returns the minimum possible thickness of the interval, given the
        uncertainty in its top and base Positions.

        Returns:
            Float: The minimum thickness.
        """
        return abs(self.base.upper - self.top.lower)

    @property
    def max_thickness(self):
        """
        Returns the maximum possible thickness of the interval, given the
        uncertainty in its top and base Positions.

        Returns:
            Float: The maximum thickness.
        """
        return abs(self.base.lower - self.top.upper)

    @property
    def kind(self):
        """
        The type of Interval: a 'point' (where base = top),
        or an 'interval', where thickness > 0.

        Returns:
            str: Either 'point' or 'interval'.
        """
        if self.thickness == 0:
            return 'point'
        return 'interval'

    @property
    def order(self):
        """
        Gives the order of this interval, based on relative values of
        top & base.
        """
        if self.top.z > self.base.z:
            return 'elevation'
        else:
            return 'depth'

    def summary(self, fmt=None, initial=False):
        """
        Returns a summary of the interval.

        Args:
            fmt (str): A format string. Optional.
            initial (bool): Whether to capitalize the first letter.

        Returns:
            str: An English-language summary.

        TODO:
            Allow formatting of the entire string, not just the rock.
        """
        s = [c.summary(fmt=fmt, initial=initial)
             for c in self.components]
        summary = " with ".join(s)
        if summary:
            return "{0:.2f} {1} of {2}".format(self.thickness, self.top.units, summary)
        elif self.description:
            return "{0:.2f} {1} of {2}".format(self.thickness, self.top.units, self.description)
        else:
            return None

    def invert(self, copy=False):
        """
        Inverts the interval. If it was depth-ordered (positive numbers
        increasing downwards.), it will now be elevation-ordered, and
        vice versa.

        Args:
            copy (bool): Whether to make a copy or not. Default: False.
        """
        if copy:
            d = self.__dict__.copy()
            del(d['top'])
            del(d['base'])
            self.base.invert()
            self.top.invert()
            return Interval(top=self.base, base=self.top, **d)
        else:
            self.base.invert()
            self.top.invert()
            old_base = self.base
            self.base = self.top
            self.top = old_base
            return

    def copy(self):
        """
        Returns a shallow copy of the interval.

        """
        return Interval(**self.__dict__.copy())

    def relationship(self, other):
        """
        Returns the relationship style. Completely deterministic.

        """
        o = {'depth': operator.lt, 'elevation': operator.gt}[self.order]
        top_inside = o(self.top.z, other.top.z) and o(other.top.z, self.base.z)
        base_inside = o(self.top.z, other.base.z) and o(other.base.z, self.base.z)
        above_below = o(other.top.z, self.top.z) and o(self.base.z, other.base.z)

        if top_inside and base_inside:
            return 'contains'
        elif above_below:
            return 'containedby'
        elif top_inside or base_inside:
            return 'partially'
        elif (self.top.z == other.base.z) or (self.base.z == other.top.z):
            return 'touches'
        else:
            return None

    def _overlaps(self, other, rel='any'):
        """
        Checks to see if and how two intervals overlap.

        """
        overlaps = ['partially', 'contains', 'containedby']
        acceptable = overlaps + ['touches', 'any']
        if rel not in acceptable:
            m = 'rel must be one of {}'.format(', '.join(acceptable))
            raise IntervalError(m)

        r = self.relationship(other)
        if r:
            if (r == rel) or ((rel == 'any') and (r in overlaps)):
                return True
        return False

    # Curry _overlaps() into some convenient functions.
    any_overlaps = partialmethod(_overlaps, rel='any')
    partially_overlaps = partialmethod(_overlaps, rel='partially')
    completely_contains = partialmethod(_overlaps, rel='contains')
    is_contained_by = partialmethod(_overlaps, rel='containedby')
    touches = partialmethod(_overlaps, rel='touches')

    def spans(self, d):
        """
        Determines if depth d is within this interval.

        Args:
            d (float): Level or 'depth' to evaluate.

        Returns:
            bool. Whether the depth is in the interval.
        """
        o = {'depth': operator.le, 'elevation': operator.ge}[self.order]
        return (o(d, self.base.z) and o(self.top.z, d))

    def split_at(self, d):
        """
        Splits an interval.

        Args:
            d (float): Level or 'depth' to split at.

        Returns:
            tuple. The two intervals that result from the split.
        """
        if not self.spans(d):
            m = 'd = {} must be within interval {}'.format(d, self)
            raise IntervalError(m)

        int1, int2 = self.copy(), self.copy()

        int1.base = d
        int2.top = d

        return int1, int2  # upper, lower

    def _explode(self, other):
        """
        Private function. 'Explodes' an interval with another interval.
        Note that `self` must at least partially overlap `other`.

        Args:
            other (Interval): The other Interval.

        Returns:
            tuple. Three Intervals: upper, middle, lower; `middle` has the
                properties of the lowermost Interval.
        """
        if not self.order == other.order:
            m = 'self and other must have the same wayupness'
            raise IntervalError(m)

        uppermost = max(self, other).copy()
        lowermost = min(self, other).copy()  # Only based on tops.

        if self.partially_overlaps(other):
            upper, _ = uppermost.split_at(lowermost.top.z)
            middle, lower = lowermost.split_at(uppermost.base.z)
        else:
            upper_temp, lower = uppermost.split_at(lowermost.base.z)
            upper, _ = upper_temp.split_at(lowermost.top.z)
            middle = lowermost

        return upper, middle, lower  # middle has lowermost's properties

    def _combine_data(self, other):
        """
        Combines data only.

        Args:
            other (Interval): The other Interval.

        Returns:
            dict. The blended data.
        """
        self_data = getattr(self, 'data', None)
        other_data = getattr(other, 'data', None)

        if (self_data is None) and (other_data is None):
            return {}
        elif (self_data is not None) and (other_data is None):
            return self_data
        elif (self_data is None) and (other_data is not None):
            return other_data
        else:
            data = {}
            for k, v in other_data.items():
                if k in self_data:
                    v = utils.list_and_add(self_data[k], v)
                data[k] = v
            return data

    def _blend_descriptions(self, other):
        """
        Private method. Computes the description for combining two intervals.
        Make sure that the intervals are already adjusted to the correct
        thicknesses.

        Args:
            other (Interval): The other Interval.

        Returns:
            str. The blended description.
        """
        thin, thick = sorted([self, other], key=lambda k: k.thickness)
        total = thin.thickness + thick.thickness
        prop = 100 * thick.thickness / total

        if self.components == other.components:
            return self.description.strip(' .,')

        d1 = thick.description.strip(' .,') or thick.summary()
        d2 = thin.description.strip(' .,') or thin.summary()
        if d1:
            d = '{:.1f}% {} with {:.1f}% {}'.format(prop, d1, 100-prop, d2)
        else:
            d = ''

        return d

    def _combine(self, old_self, other, blend=True):
        """
        Private method. Combines data, components, and descriptions but
        nothing else.

        Args:
            old_self (Interval): You have to pass the instance explicitly.
            other (Interval): The other Interval.
            blend (bool): Whether to blend or not.

        Returns:
            Interval. The combined description.
        """
        if blend:
            self.components = old_self.components.copy()
            for c in other.components:
                if c not in self.components:
                    self.components.append(c)
            self.description = old_self._blend_descriptions(other)
            self.data = old_self._combine_data(other)
        else:
            self.components = other.components
            self.description = other.description
            self.data = other.data

        return self

    def intersect(self, other, blend=True):
        """
        Perform the intersection binary operation. self must at least
        partially overlap with other or an IntervalError is raised.

        If blend is False, you are essentially replacing self with other.

        Args:
            other (Interval): The other Interval.
            blend (bool): Whether to blend or not.

        Returns:
            Interval. The intersection of the Interval with the one provided.
        """
        if not self.any_overlaps(other):
            m = 'self must at least partially overlap other'
            raise IntervalError(m)

        _, intersection, _ = self._explode(other)

        return intersection._combine(self, other, blend=blend)

    def merge(self, other, blend=True):
        """
        Perform the merge binary operation. self must at least
        partially overlap with other or an IntervalError is raised.

        If blend is False, you are essentially replacing self with other.

        Args:
            other (Interval): The other Interval.
            blend (bool): Whether to blend or not.

        Returns:
            Striplog. The merge of the Interval with the one provided.
        """
        if not self.any_overlaps(other):
            m = 'self must at least partially overlap other'
            raise IntervalError(m)

        upper, middle, lower = self._explode(other)

        if self.top.z == upper.top.z:
            self_is_uppermost = True
        else:
            self_is_uppermost = False

        middle = middle._combine(self, other, blend=blend)

        if self.partially_overlaps(other) and (not blend):
            # Then we'll only have two pieces:
            if self_is_uppermost:
                result = [upper, other]
            else:
                result = [other, lower]
        else:
            result = [lower, middle, upper]

        from .striplog import Striplog  # Import here to avoid circular ref
        if self.order == 'depth':
            return Striplog(result[::-1])
        else:
            return Striplog(result)

    def union(self, other, blend=True):
        """
        Perform the union binary operation. self must at least touch other or
        an IntervalError is raised.

        If blend is False, you are essentially replacing self with other.

        Args:
            other (Interval): The other Interval.
            blend (bool): Whether to blend or not.

        Returns:
            Interval. The union of the Interval with the one provided.
        """
        if not (self.touches(other) or self.any_overlaps(other)):
            # m = 'self must at least touch or partially overlap other'
            # raise IntervalError(m)
            return self, other

        if self.order == 'elevation':
            top = max(self.top.z, other.top.z)
            bot = min(self.base.z, other.base.z)
        else:
            top = min(self.top.z, other.top.z)
            bot = max(self.base.z, other.base.z)

        result = self.copy()
        result.top = top
        result.base = bot

        return result._combine(self, other, blend=blend)

    def difference(self, other):
        """
        Perform the difference binary operation.

        Args:
            other (Interval): The other Interval.

        Returns:
            Interval. One or two Intervals.
        """
        if self.touches(other) or (not self.any_overlaps(other)):
            return self
        elif self.completely_contains(other):
            upper, _, lower = self._explode(other)
            return upper, lower
        else:
            if self > other:
                return self.split_at(other.top.z)[0]
            elif self < other:
                return self.split_at(other.base.z)[1]
            else:  # They are equal
                return None

    @staticmethod
    def _parse_description(description, lexicon,
                            max_component=1,
                            abbreviations=False):
        """
        Turns a description into a lists of components. The items in the
        list are in the order they were found in the description, which is
        usually order of importance.

        Args:
            lexicon (Lexicon): The translation between words and their meaning.
            max_component (int): The most components to return. Default 1.
            abbreviations (bool): Whether to expand abreviations or not.
                Default False.

        Returns:
            List. A list of Components extracted from the description text.
        """

        if abbreviations:
            text = lexicon.expand_abbreviations(description)
        else:
            text = description

        components = []
        for p, part in enumerate(lexicon.split_description(text)):
            if p == max_component:
                break
            components.append(Component.from_text(part, lexicon))

        return components


text:
package org.apache.samza.metrics;

public interface ReadableMetricsRegistryListener {
  void onCounter(String group, Counter counter);

  void onGauge(String group, Gauge<?> gauge);

  void onListGauge(String group, ListGauge<?> listGauge);

  void onTimer(String group, Timer timer);
}


text:
package com.airbnb.airmapview.sample;

import android.content.Context;
import android.util.Log;

import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.StringWriter;
import java.io.Writer;

public class Util {

  private static final String LOGTAG = "Util";

  public static String readFromRawResource(Context context, int resourceId) {
    InputStream resourceReader = context.getResources().openRawResource(resourceId);
    Writer writer = new StringWriter();
    try {
      BufferedReader reader = new BufferedReader(new InputStreamReader(resourceReader, "UTF-8"));
      String line = reader.readLine();
      while (line != null) {
        writer.write(line);
        line = reader.readLine();
      }
    } catch (Exception e) {
      Log.e(LOGTAG, "Unhandled exception while reading from resource", e);
    } finally {
      try {
        resourceReader.close();
      } catch (Exception e) {
        Log.e(LOGTAG, "Unhandled exception while closing resource", e);
      }
    }

    return writer.toString();
  }

}


text:
layout: page
title: Ferguson Family Reunion
date: 2016-05-24
author: Katherine Copeland
tags: weekly links, java
status: published
summary: Proin felis ex, hendrerit vitae mollis non.
banner: images/banner/meeting-01.jpg
booking:
  startDate: 02/26/2019
  endDate: 03/02/2019
  ctyhocn: HOUHBHX
  groupCode: FFR
published: true
---
Nam faucibus arcu sit amet velit ultricies, sit amet porttitor mauris rutrum. Quisque hendrerit justo accumsan, auctor augue vel, dictum lacus. Proin vel volutpat justo, interdum tristique nisl. Duis in justo ut mauris ultricies luctus. In lacinia elit id efficitur posuere. Cras imperdiet libero sit amet erat gravida, non fermentum elit porttitor. Sed mollis iaculis semper. Suspendisse volutpat tincidunt magna, sit amet luctus felis suscipit vel.
Praesent porttitor quam sodales, rhoncus metus quis, maximus nisi. Donec risus sapien, lobortis at iaculis ac, euismod non leo. Curabitur sollicitudin libero id molestie semper. Aliquam erat volutpat. Vivamus facilisis nec lectus quis facilisis. Aliquam malesuada velit ac turpis tincidunt pretium. Nulla felis metus, ultrices at ante eu, iaculis auctor sem. Quisque at porttitor purus. Pellentesque eu diam ipsum. Aliquam erat volutpat. Aliquam tincidunt dui eu diam hendrerit, nec pulvinar nibh consequat. Maecenas suscipit ut lectus id efficitur. Cras aliquam sodales neque a gravida. Curabitur ultricies nisl neque, quis sollicitudin urna tempus at.

1 Pellentesque et leo pharetra, rhoncus mauris nec, lacinia dui
1 Proin sit amet est vitae eros ullamcorper lacinia consequat vel nibh.

Suspendisse scelerisque ante a consequat bibendum. Donec ac posuere nisl. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Duis et tellus et orci viverra placerat nec id est. Nulla bibendum scelerisque dignissim. Praesent commodo quis enim vel pulvinar. Mauris at ante facilisis, elementum diam at, gravida lacus. Donec ac lorem facilisis, placerat odio a, bibendum tellus. Quisque egestas eget mauris in consectetur. Duis placerat placerat ante, sit amet scelerisque tortor lacinia vitae. Curabitur vehicula felis at felis aliquam, in pulvinar nisi cursus.


text:
dotfiles-vim
============

My ~/.vim and  ~/.vimrc

## Installation

    cd ~/.config
    mkdir vim

Now download dotfiles-vim and copy its vim/ content into ~/.config/vim.

If ~/.vim exists, delete it after backing it up.

    cd ~
    ln -s .config/vim .vim
    sudo aptitude install vim-youcompleteme
    vim-addon-manager install youcompleteme
    cd .config/vim
    mkdir bundle
    cd bundle
    git clone https://github.com/Lokaltog/powerline.git
    git clone https://github.com/Shougo/neomru.vim.git
    git clone https://github.com/Shougo/unite.vim.git
    git clone https://github.com/airblade/vim-gitgutter.git
    git clone https://github.com/altercation/vim-colors-solarized.git
    git clone https://github.com/ap/vim-css-color.git
    git clone https://github.com/docunext/closetag.vim.git
    git clone https://github.com/elzr/vim-json.git
    git clone https://github.com/msanders/snipmate.vim.git
    git clone https://github.com/pangloss/vim-javascript.git
    git clone https://github.com/rking/ag.vim.git
    git clone https://github.com/scrooloose/nerdtree.git
    git clone https://github.com/scrooloose/syntastic.git
    git clone https://github.com/tpope/vim-fugitive.git
    git clone https://github.com/tpope/vim-haml.git
    git clone https://github.com/tpope/vim-pathogen.git
    git clone https://github.com/tpope/vim-sensible.git
    git clone https://github.com/tpope/vim-sleuth.git
    git clone https://github.com/tpope/vim-surround.git
    git clone https://github.com/vim-scripts/TaskList.vim.git
    git clone https://github.com/vim-scripts/taglist.vim.git

Install
[Python syntax checkers for Syntastic](https://github.com/scrooloose/syntastic/wiki/Syntax-Checkers):

    sudo aptitude install flake8 pep8 pyflakes pylint

Install [pylint-django](https://github.com/landscapeio/pylint-django).

Optional (at this moment, only available in Debian Testing, but you can use
[apt-pinning](https://wiki.debian.org/AptPreferences#Pinning)):

    sudo aptitude install silversearcher-ag


text:
This sample shows how provision sites with Yammer feed associated as the default news feed for the site. Shows options with OpenGraph objects or by using groups (existing or create new).

### Applies to ###
-  Office 365 Multi Tenant (MT)
-  Office 365 Dedicated (D)
-  SharePoint 2013 on-premises

*Since integration to the Yammer feeds actually happens with browser, this does work in the on-premises if the browsers have Internet connectivity.*

### Prerequisites ###
You will have to create Yammer access token for the app for your Yammer network. Additional details and link to get this done below in this document.

### Solution ###
Solution | Author(s)
---------| ----------
Provisioning.Yammer | Vesa Juvonen (Microsoft)

### Version history ###
Version  | Date | Comments
---------| -----| --------
1.0  | June 30th 2014 | Initial release

### Disclaimer ###
**THIS CODE IS PROVIDED *AS IS* WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING ANY IMPLIED WARRANTIES OF FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, OR NON-INFRINGEMENT.**

----------

# Create sub site and replace default news feed with Yammer Feed #
This sample shows how to use the newly added Yammer capabilities in the core component to create new Yammer group as part of site provisioning and to associate that as the discussion feed for the newly created collaborations site.

To be able to use the sample, follow guidance for this URL to register access token for your Yammer app. This access token in updated to the web.config of the provider hosted app.

- Get the access token from here: [https://developer.yammer.com/authentication](https://developer.yammer.com/authentication)

Update access token to the web.config of the provider hosted app for the key called YammerAccessToken.
```XML
<!-- Details on how to get your access token - check following https://developer.yammer.com/authentication -->
<add key="YammerAccessToken" value="PutYourOwnYammerKeyHere" />
```

Sample provides simple provisioning UI for sub sites where you are able to define if you want to use OpenGraph or Group feed. If group feed option is selected, you can choose to associate the feed to any existing group or to create a new group for the site.

----------
**Notice**. We do recommend usage of OpenGraph objects with the team site feeds rather than using groups. If you automatically provision groups for each of the created team site, you will end up having massive amount of groups which will pollute your solution story in Yammer. Usage of the REST APIs for creating groups is also not documented and could have unexpected issues.

----------

![](http://i.imgur.com/n7G6tUJ.png)

Once the provisioning is completed, you will see the following type of a site with associated Yammer feed running in it. Notice that you will need to be logged to the particular Yammer network in the browser session to show the news feed, since integration of the yammer feed happens in browser side.

![](http://i.imgur.com/NH4WMdL.png)

*Notice that by default feed is added without header or footer elements from the Yammer, but that can be controlled from the code which is calling the embed capability.*

This is how the UI looks if you have not signed to the Yammer network. You can click the login button, which will make the yammer feed to work again as expected.

![](http://i.imgur.com/ITVpzXL.png)

This example creates new Yammer group for each of the team site. We could actually also create new OpenGraph objects if that's preferred. Either way the group or OpenGraph object is visible also on the Yammer side as follows and any updates from either side is updated to the feed. Here's what gets created to the Yammer side as result of the site provisioning.

![](http://i.imgur.com/FwMqzxY.png)

Notice also that since this configuration is dynamically applied during provisioning time, there's no impact on removing the provisioning app away from the SharePoint side.

# Used Core component extensions #

Actual provisioning logic and site modifications are using [PnP Core component](https://github.com/OfficeDev/PnP/tree/master/OfficeDevPnP.Core) extension methods. As you can see we can perform the required actions with only few lines of code due the encapsulated reusable methods from the core component.

```C#
public void CreateSubSite(Web hostWeb, string url, string template,
                            string title, string description, string feedType, string yammerGroupName)
{
    // Create new sub site
    Web newWeb = hostWeb.CreateWeb(title, url, description, template, 1033);

    //Remove the out of the box "NewsFeed" web part
    newWeb.DeleteWebPart("SitePages", "Site feed", "home.aspx");

    // Let's first get the details on the Yammer network using the access token
    WebPartEntity wpYammer;
    YammerUser user = YammerUtility.GetYammerUser(ConfigurationManager.AppSettings["YammerAccessToken"]);

    // Created Yammer web part with needed configuration
    wpYammer = CreateYammerWebPart(feedType, user, yammerGroupName, title);

    // Add Yammer web part to the page
    newWeb.AddWebPartToWikiPage("SitePages", wpYammer, "home.aspx", 2, 1, false);

    // Add theme to the site and apply that
    ApplyThemeToSite(hostWeb, newWeb);
}
```


text:
.. _install_sandboxes_jaeger_native_tracing:

Jaeger Native Tracing
=====================

The Jaeger tracing sandbox demonstrates Envoy's :ref:`request tracing <arch_overview_tracing>`
capabilities using `Jaeger <http://jaegertracing.io/>`_ as the tracing provider and Jaeger's native
`C++ client <https://github.com/jaegertracing/jaeger-client-cpp>`_ as a plugin. Using Jaeger with its
native client instead of with Envoy's builtin Zipkin client has the following advantages:

- Trace propagation will work with the other services using Jaeger without needing to make
  configuration `changes <https://github.com/jaegertracing/jaeger-client-go#zipkin-http-b3-compatible-header-propagation>`_.
- A variety of different `sampling strategies <https://www.jaegertracing.io/docs/sampling/#client-sampling-configuration>`_
  can be used, including probabilistic or remote where sampling can be centrally controlled from Jaeger's backend.
- Spans are sent to the collector in a more efficient binary encoding.

This sandbox is very similar to the front proxy architecture described above, with one difference:
service1 makes an API call to service2 before returning a response.
The three containers will be deployed inside a virtual network called ``envoymesh``. (Note: the sandbox
only works on x86-64).

All incoming requests are routed via the front envoy, which is acting as a reverse proxy
sitting on the edge of the ``envoymesh`` network. Port ``80`` is mapped to  port ``8000``
by docker compose (see :repo:`/examples/jaeger-native-tracing/docker-compose.yml`). Notice that
all envoys are configured to collect request traces (e.g., http_connection_manager/config/tracing setup in
:repo:`/examples/jaeger-native-tracing/front-envoy-jaeger.yaml`) and setup to propagate the spans generated
by the Jaeger tracer to a Jaeger cluster (trace driver setup
in :repo:`/examples/jaeger-native-tracing/front-envoy-jaeger.yaml`).

Before routing a request to the appropriate service envoy or the application, Envoy will take
care of generating the appropriate spans for tracing (parent/child context spans).
At a high-level, each span records the latency of upstream API calls as well as information
needed to correlate the span with other related spans (e.g., the trace ID).

One of the most important benefits of tracing from Envoy is that it will take care of
propagating the traces to the Jaeger service cluster. However, in order to fully take advantage
of tracing, the application has to propagate trace headers that Envoy generates, while making
calls to other services. In the sandbox we have provided, the simple flask app
(see trace function in :repo:`/examples/front-proxy/service.py`) acting as service1 propagates
the trace headers while making an outbound call to service2.

Running the Sandbox
~~~~~~~~~~~~~~~~~~~

The following documentation runs through the setup of an envoy cluster organized
as is described in the image above.

**Step 1: Build the sandbox**

To build this sandbox example, and start the example apps run the following commands::

    $ pwd
    envoy/examples/jaeger-native-tracing
    $ docker-compose up --build -d
    $ docker-compose ps
            Name                       Command               State      Ports
    -------------------------------------------------------------------------------------------------------------
    jaegertracing_service1_1      /bin/sh -c /usr/local/bin/ ... Up       80/tcp
    jaegertracing_service2_1      /bin/sh -c /usr/local/bin/ ... Up       80/tcp
    jaegertracing_front-envoy_1   /bin/sh -c /usr/local/bin/ ... Up       0.0.0.0:8000->80/tcp, 0.0.0.0:8001->8001/tcp

**Step 2: Generate some load**

You can now send a request to service1 via the front-envoy as follows::

    $ curl -v $(docker-machine ip default):8000/trace/1
    *   Trying 192.168.99.100...
    * Connected to 192.168.99.100 (192.168.99.100) port 8000 (#0)
    > GET /trace/1 HTTP/1.1
    > Host: 192.168.99.100:8000
    > User-Agent: curl/7.43.0
    > Accept: */*
    >
    < HTTP/1.1 200 OK
    < content-type: text/html; charset=utf-8
    < content-length: 89
    < x-envoy-upstream-service-time: 1
    < server: envoy
    < date: Fri, 26 Aug 2016 19:39:19 GMT
    < x-envoy-protocol-version: HTTP/1.1
    <
    Hello from behind Envoy (service 1)! hostname: f26027f1ce28 resolvedhostname: 172.19.0.6
    * Connection #0 to host 192.168.99.100 left intact

**Step 3: View the traces in Jaeger UI**

Point your browser to http://localhost:16686 . You should see the Jaeger dashboard.
Set the service to "front-proxy" and hit 'Find Traces'. You should see traces from the front-proxy.
Click on a trace to explore the path taken by the request from front-proxy to service1
to service2, as well as the latency incurred at each hop.


text:
<?php

class Google_Service_HomeGraphService_QueryRequest extends Google_Collection
{
  protected $collection_key = 'inputs';
  public $agentUserId;
  protected $inputsType = 'Google_Service_HomeGraphService_QueryRequestInput';
  protected $inputsDataType = 'array';
  public $requestId;

  public function setAgentUserId($agentUserId)
  {
    $this->agentUserId = $agentUserId;
  }
  public function getAgentUserId()
  {
    return $this->agentUserId;
  }
  /**
   * @param Google_Service_HomeGraphService_QueryRequestInput
   */
  public function setInputs($inputs)
  {
    $this->inputs = $inputs;
  }
  /**
   * @return Google_Service_HomeGraphService_QueryRequestInput
   */
  public function getInputs()
  {
    return $this->inputs;
  }
  public function setRequestId($requestId)
  {
    $this->requestId = $requestId;
  }
  public function getRequestId()
  {
    return $this->requestId;
  }
}


text:
module Parser.Types
  ( Parser
  , lexeme
  ) where

import           Control.Applicative
import           Data.Text                  (Text)
import           Data.Void                  (Void)

import           Text.Megaparsec
import           Text.Megaparsec.Char
import qualified Text.Megaparsec.Char.Lexer as L

type Parser = Parsec Void Text

-- | Defines how our parsers will consume space characters. You can use the
--   second and third arguments to define consumers for line/block comments,
--   but we're not parsing a full language here, so we don't need them.
sc :: Parser ()
sc = L.space space1 empty empty

-- | Used to wrap parsers in our space consumer so they will automatically
--   consume that space without explicitly needing to state it.
lexeme :: Parser a -> Parser a
lexeme = L.lexeme sc


text:
Swatches
========

Demonstrates using Multipeer Connectivity with Apple Watch.

Running
=======

1. Run the ```Swatches Watch App``` target, which will launch the simulator
2. If the watch simulator isnt't enabled you can turn it on under Hardware -> External Displays
3. Run the host app on the phone simulator, which the watch app will connect to
4. Tapping any of the colored buttons should change the host app's background view color


text:
package org.springframework.boot.autoconfigure.data.redis;

import org.springframework.boot.autoconfigure.AutoConfigureAfter;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.repository.configuration.EnableRedisRepositories;
import org.springframework.data.redis.repository.support.RedisRepositoryFactoryBean;

/**
 * {@link EnableAutoConfiguration Auto-configuration} for Spring Data's Redis
 * Repositories.
 *
 * @author Eddú Meléndez
 * @author Stephane Nicoll
 * @see EnableRedisRepositories
 * @since 1.4.0
 */
@Configuration
@ConditionalOnClass(EnableRedisRepositories.class)
@ConditionalOnBean(RedisConnectionFactory.class)
@ConditionalOnProperty(prefix = "spring.data.redis.repositories", name = "enabled", havingValue = "true", matchIfMissing = true)
@ConditionalOnMissingBean(RedisRepositoryFactoryBean.class)
@Import(RedisRepositoriesAutoConfigureRegistrar.class)
@AutoConfigureAfter(RedisAutoConfiguration.class)
public class RedisRepositoriesAutoConfiguration {

}


text:
<template name="oembedImageWidget">
	{{#if parsedUrl}}
		{{#if collapsed}}

		{{else}}

			{{#if loadImage}}

			{{else}}

click to load

			{{/if}}
		{{/if}}
	{{/if}}
</template>