<file_sep>#include <stdio.h>
#include <stdlib.h>
#include "common.c"

unsigned char curbyte = 0;
char bitsleft = 0;

char nextbit() {
	//returns 0 or 1
	//<0 for error
	if (bitsleft == 0) {
		int in=getchar();
		if (in == EOF) return(-1);
		curbyte = in;
		bitsleft = 8;
	}
	//printf("curbtye: %hhu\n", curbyte);
	char ret = (curbyte&0x80)>0;
	curbyte<<=1;
	bitsleft--;
	return ret;
}

int main() {
	char inbits[2] = {0, 0};
	while ((inbits[0]=nextbit())>=0 && (inbits[1]=nextbit())>=0) {
		//core of the von neumann filter
		//printf("inbits: %hhd%hhd\n", inbits[0], inbits[1]);
		if (inbits[0] == inbits[1]) continue;
		else buildbyte(inbits[0]);
	}
	return(0);
}
<file_sep>/********************************/
/*  <NAME> 2008-2009        */
/*  v1.0                        */
/*  4/20/2009                   */
/********************************/

int baud_rate = 19200;
int adc_pin = 0;

byte partbyte = 0;
byte bitcount = 0;

void setup(){
  Serial.begin(baud_rate);
  //analogReference(EXTERNAL);
}

void loop(){
  int adc_value = analogRead(adc_pin);
//  Serial.println(adc_value, DEC);
//  return;
  boolean bit_in = adc_value % 2;
  buildbyte(bit_in);
  //delay(10);
}

void buildbyte(boolean bit_in) {
  //we don't have to initialize partbyte
  partbyte = (partbyte << 1) + bit_in;
  bitcount = (bitcount + 1) % 8;
  if (bitcount == 0) {
    Serial.print(partbyte, BYTE);
  }
}

<file_sep>#include <stdio.h>
#include <stdlib.h>

int readblock(long blocksize, char* buf){
	//will return 1 when the block has not been filled
	//unfilled blocks (the last one, usually) are ignored
	//returns 0 on a successfully filled block
	int i=0;
	int in=0;
	for (i=0; i<blocksize; i++) {
		in = getchar();
		if (in == EOF) return(1);
		buf[i] = in;
	}
	return(0);
}

char partbyte = 0;
char numbits = 0;

void buildbyte(char in_bit) {
	//in_bit should be 0 or 1
	partbyte = (partbyte << 1) + in_bit;
	numbits = (numbits + 1) % 8;
	if (numbits == 0){
		putchar(partbyte);
	}
}
<file_sep>#!/usr/bin/python2
import serial, sys

try:
    chunksize = int(sys.argv[1])
except IndexError:
    chunksize = 4096

stdout = sys.stdout

s = serial.Serial('/dev/ttyUSB0', 19200)

while 1:
	buf = s.read(chunksize)
	if not buf:
		break
	stdout.write(buf)
	stdout.flush()
<file_sep>#!/bin/bash

#test each script:
#raw2bin.py
#bin2hist.py -s 2048
#bin2bias.py -s 2048
#hist2pm3d.py
#C scripts as well!
#compare md5sums of good to bad

function checksum {
	CHECK=$(md5sum $1 | cut -b-32)
	GOOD=$(md5sum $1.good | cut -b-32)
	if [[ $CHECK = $GOOD ]]; then 
		echo "OK: $1"
	else
		echo "!!! BAD !!!: $1"
	fi
}

function status {
	echo -ne "$1...\033[0K\r"
}

if [[ $0 != "./test" ]]; then
	echo "Run me from my directory!"
	exit
fi

echo "Compiling C binaries..."
gcc ../raw2bin.c	-o ../raw2bin
gcc ../bin2bias.c	-o ../bin2bias

echo "Checking C binaries:"
status raw2bin
cat raw-test.good | ../raw2bin > sample-raw-test
checksum sample-raw-test
status vnf
cat sample-raw-test | ../vnf > sample-raw-test-vnf
checksum sample-raw-test-vnf
status bin2bias
cat sample-raw-test | ../bin2bias 2048 > sample-raw-test.bias
checksum sample-raw-test.bias

echo "Checking python scripts:"
status raw2bin.py
cat raw-test.good | ../raw2bin.py > sample-raw-test
checksum sample-raw-test
status vnf.py
cat sample-raw-test | ../vnf.py > sample-raw-test-vnf
checksum sample-raw-test-vnf
status bin2bias.py
cat sample-raw-test | ../bin2bias.py 2048 > sample-raw-test.bias
checksum sample-raw-test.bias
status bin2hist.py
cat sample-raw-test | ../bin2hist.py 2048 > sample-raw-test.hist
checksum sample-raw-test.hist
status hist2pm3d.py
cat sample-raw-test.hist | ../hist2pm3d.py > sample-raw-test.hist-cov
checksum sample-raw-test.hist-cov
<file_sep>#!/usr/bin/python

import sys

global partbit, numbits
partbit = 0
numbits = 0

def buildbyte(bit_in):
    global partbit, numbits
    partbit = (partbit << 1) + bit_in
    numbits = (numbits + 1) % 8
    #print("BIT_IN", bit_in, file=sys.stderr)
    if numbits == 0:
        #this weird line stops stdout from unicoding the bytes
        sys.stdout.write(bytes(chr(partbit), "latin-1"))
        partbit = 0 

global curbyte, bitsleft
curbyte = 0
bitsleft = 0

def nextbit():
    global curbyte, bitsleft
    if not bitsleft:
        curbyte = sys.stdin.read(1)
        if not curbyte: return -1
        curbyte = ord(curbyte)
        bitsleft = 8
    #print(curbyte, file=sys.stderr)
    ret = int((curbyte&0x80)>0)
    curbyte = (curbyte<<1) & 0xff
    bitsleft -= 1
    return ret

sys.stdout = sys.stdout.detach()
sys.stdin = sys.stdin.detach()

while 1:
    inbits = [nextbit(), nextbit()]
    #print(inbits, file=sys.stderr)
    if -1 in inbits: break
    if inbits[0] != inbits[1]:
        buildbyte(inbits[0])

<file_sep>#include <stdio.h>
#include <stdlib.h>
#include "common.c"

//for values 0-1023, this should be fine at 5
#define MAXLINELENGTH 10

int readline(char* buf) {
	int i=0;
	int in=0;
	for (i=0; i<MAXLINELENGTH; i++) {
		in = getchar();
		if (in == EOF) return(1);
		if (in == '\n') {
			buf[i] = 0;
			return(0);
		}
		buf[i] = in;
	}
	//if we get to here, we exceeded MAXLINELENGTH
	return(2);
}

int main() {
	char buf[MAXLINELENGTH];
	while (readline(buf) == 0) {
		//printf("%s\n", buf);
		buildbyte(atoi(buf)%2);
	}
	return(0);
}
<file_sep>#!/usr/bin/python
import sys, argparse

#parser = argparse.ArgumentParser(description='Analyze bitwise bias of \
#binary data', epilog="Input is binary data for analysis. Output is lines \
#with place in file on vertical axis like this: #0 #1 %0 %1")
#parser.add_argument('-s', default=4096, dest='chunksize', type=int,
#                   help='size of each chuck to consider separately')

#chunksize = parser.parse_args().chunksize
try:
    chunksize = int(sys.argv[1])
except IndexError:
    chunksize = 4096

stdin = sys.stdin.detach()

while 1:
	chunk = stdin.read(chunksize)
	if len(chunk) < chunksize:
		break
	counts = [0, 0]
	for byte in chunk:
#		bits = bin(byte)[2:]
#		for bit in "0"*(8-len(bits))+bits:
#			counts[int(bit)] += 1
		for bit in [int(bool(byte & x)) for x in [0x80, 0x40, 0x20, 0x10, 0x8, 0x4, 0x2, 0x1]]:
			counts[int(bit)] += 1
	p0 = counts[0]/sum(counts)
	p1 = counts[1]/sum(counts)
	print('%d %d %f %f' % (counts[0], counts[1], p0, p1))
<file_sep>#!/usr/bin/python
import sys, subprocess
from queue import Queue
from threading import Thread, Lock
from time import sleep

#basically an application-wide buffer size
#this many bytes are handled at a time
#up to this many bytes may be dropped when using tap()
#and directing it to not copy the data (the default)
BLOCKSIZE=128


tostr = lambda byte: str(byte, 'latin-1')
tobytes = lambda string: bytes(string, 'latin-1')
def totype(indata, targettype):
    if type(indata) == targettype: return indata
    if targettype == str:       return tostr(indata)
    elif targettype == bytes:   return tobytes(indata)

def run(command):
    "takes command and arguments in a list"
    "returns stdout and stdin file objects"
    p = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    return(p.stdout, p.stdin, p)


class StreamNode():
    "A running command which is a piece of the data pipe"
    def __init__(self, command, types):
        self.stdout, self.stdin, self.p = run(command)
        self.outtype, self.intype = types
    
    def stop():
        self.p.kill()

class EndNode():
    "A file which is only written to and forms the end of a pipe"
    def __init__(self, infile, datatype=str):
        self.stdout, self.stdin = None, infile
        self.outtype, self.intype = None, datatype 
        self.p = None
    
    def stop():
        pass

class SourceNode():
    "A file which is only read from and forms the beginning of the pipe"
    def __init__(self, outfile, datatype=str):
        self.stdout, self.stdin = outfile, None
        self.outtype, self.intype = datatype, None
        self.p = None
    
    def stop():
        pass


class SplitThread(Thread):
    "grabs input from an infile and duplicates it across one or more outs"
    "can be tapped in both yank and del modes"
    "file descriptors used by this class must be in BINARY MODE"
    def init(self, infile, outfiles, name=None):
        self.setDaemon(True)
        self.infile, self.outfiles = infile, outfiles
        self.tapreqs = Queue()
        self.name = name
        self.debug = False

    def tap(self, n, copy=False):
        "grab n bytes from the running pipe"
        "if copy is False, the bytes returned (plus enough to make an even"
        "block) will be removed from the pipe and never seen by outfiles"
        "returns bytes"
        "this method SHOULD be threadsafe"
        "name is used for identifying when we error out"
        nblocks = n//BLOCKSIZE+1
        blocks = []
        for bid in range(nblocks):
            q = Queue()
            self.tapreqs.put([copy, q])
            blocks.append(q.get())
        return b''.join([totype(b, bytes) for b in blocks])[:n]
    
    def addout(self, out):
        "add an out to the splitter using this method"
        "argument is in standard format of [fdesc, type]"
        "to remove an out, simply close the file descriptor"
        self.outfiles.append(out)

    def run(self):
        while 1:
            try:
                block = self.infile.read(BLOCKSIZE)
                if len(block) < BLOCKSIZE:
                    raise IOError("Block not filled")
            except:
                #what do we do now? the command exited, probably
                #in any case, we can't go on
                #TODO: investigate what happens to downstream stuff
                print("SplitThread's infile died (name=%s)" % self.name)
                print(sys.exc_info())
                for out in self.outfiles:
                    out[0].close()
                break
            if self.tapreqs.qsize():
                copy, cbq = self.tapreqs.get()
                cbq.put(block)
                if self.debug:
                    print("SplitThread %s giving block to tap" % self.name)
                if not copy:
                    continue
            if self.debug:
                print("SplitThread %s sending block to outfiles" % self.name)
            for out in self.outfiles:
                try:
                    out[0].write(totype(block, out[1]))
                    out[0].flush()
                except:
                    print("Removing outfile from SplitThread (name=%s):" % self.name)
                    print(sys.exc_info())
                    self.outfiles.remove(out)

def linknodes(innode, outnodes, name):
    infile = innode.stdout
    outfiles = []
    for node in outnodes:
        outfiles.append([node.stdin, node.intype])
    splitter = SplitThread()
    splitter.init(infile, outfiles, name)
    splitter.start()
    return splitter 

def addnode(splitter, outnode):
    splitter.addout([node.stdin, node.intype])


if __name__ == "__main__":
    #dumpn = StreamNode(['cat', '/dev/urandom'], [bytes, None])
    dumpn = StreamNode(['./dump.py', str(BLOCKSIZE)], [bytes, None])
    #dumpn = SourceNode(open('server.py', 'r'))
    #hexn = StreamNode(['hexdump', '-C'], [bytes, bytes])
    vnfn = StreamNode('./vnf', [bytes, bytes])
    #printn = EndNode(sys.stdout)
    dumps = linknodes(dumpn, [vnfn], "dump")
    dumps.debug = True
    vnfs = linknodes(vnfn, [], "vnf")
    vnfs.debug = True
    #hexs = linknodes(hexn, [printn], "hex > print")
    while 1:
        sleep(1)
        print("dumps", dumps.tap(100, True))
        print("vnfs", vnfs.tap(100, True))
        #i = dumps.tap(100, False)
        #this makes a mess
        #print(i)


<file_sep>#!/usr/bin/python

import sys

sys.stdout = sys.stdout.detach()
partbit = 0
numbits = 0

while 1:
	line = sys.stdin.readline()
	if not line: break
	bit_in = int(line) % 2
	partbit = (partbit << 1) + bit_in
	numbits = (numbits + 1) % 8
	if numbits == 0:
		#this weird line stops stdout from unicoding the bytes
		sys.stdout.write(bytes(chr(partbit), "latin-1"))
		partbit = 0 
<file_sep>#include <stdio.h>
#include <stdlib.h>
#include "common.c"

void checkblock(long blocksize, char* buf) {
	int cid=0;
	int mask=0;
	char comp=1;
	long counts[2] = {0, 0};
	for (cid=0; cid<blocksize; cid++) {
		for (mask=1; mask<=128; mask=mask << 1) {
			if ((buf[cid]&mask)>0) { counts[1]++; }
			else { counts[0]++; }
		}
	}
	double sum = counts[0] + counts[1];
	float p0 = counts[0]/sum;
	float p1 = counts[1]/sum;
	printf("%ld %ld %f %f\n", counts[0], counts[1], p0, p1);
}

int main(int argc, char *argv[]) {
	int i = 0; //currently unused
	long blocksize = 4096;
	if (argc > 1) {
		blocksize = atol(argv[1]);
	}
	char* buf = malloc(blocksize);
	while (readblock(blocksize, buf) == 0) {
		checkblock(blocksize, buf);
	}
	free(buf);
	return(0);
}

<file_sep>#!/usr/bin/python
import sys, argparse

#parser = argparse.ArgumentParser(description='Analyze bitwise bias of \
#binary data', epilog="Input is binary data for freq analysis. Output is \
#space-separated lines of waterfall-like quantites with place in file on \
#vertical axis, byte values on horizontal axis, and popularity of each byte \
#as values.")
#parser.add_argument('-s', default=4096, dest='chunksize', type=int,
#                   help='size of each chuck to consider separately')
#chunksize = parser.parse_args().chunksize
try:
    chunksize = int(sys.argv[1])
except IndexError:
    chunksize = 4096

stdin = sys.stdin.detach()

while 1:
	chunk = stdin.read(chunksize)
	if len(chunk) < chunksize:
		break
	counts = {}
	for i in range(256): counts[i] = 0
	for byte in chunk:
		counts[byte] += 1
	for i in range(256):
		sys.stdout.write(' %d' % counts[i])
	print()
<file_sep>#!/usr/bin/python
import sys

#input should be space-separated lines of
#waterfall-like quantites
#with time on the vertical axis
#types on horizontal axis
#bar heights as values

linecnt = 0

while 1:
	line = sys.stdin.readline()
	if len(line) == 0:
		break
	colcnt  = 0
	for col in line.split():
		print('%d %d %d' % (linecnt, colcnt, float(col)))
		colcnt += 1
	print()
	linecnt += 1
