text:
var async = require('../lib');
var expect = require('chai').expect;
var assert = require('assert');

describe('queue', function(){

    it('basics', function(done) {

        var call_order = [];
        var delays = [40,20,60,20];

        // worker1: --1-4
        // worker2: -2---3
        // order of completion: 2,1,4,3

        var q = async.queue(function (task, callback) {
            setTimeout(function () {
                call_order.push('process ' + task);
                callback('error', 'arg');
            }, delays.shift());
        }, 2);

        q.push(1, function (err, arg) {
            expect(err).to.equal('error');
            expect(arg).to.equal('arg');
            expect(q.length()).to.equal(1);
            call_order.push('callback ' + 1);
        });
        q.push(2, function (err, arg) {
            expect(err).to.equal('error');
            expect(arg).to.equal('arg');
            expect(q.length()).to.equal(2);
            call_order.push('callback ' + 2);
        });
        q.push(3, function (err, arg) {
            expect(err).to.equal('error');
            expect(arg).to.equal('arg');
            expect(q.length()).to.equal(0);
            call_order.push('callback ' + 3);
        });
        q.push(4, function (err, arg) {
            expect(err).to.equal('error');
            expect(arg).to.equal('arg');
            expect(q.length()).to.equal(0);
            call_order.push('callback ' + 4);
        });
        expect(q.length()).to.equal(4);
        expect(q.concurrency).to.equal(2);

        q.drain = function () {
            expect(call_order).to.eql([
                'process 2', 'callback 2',
                'process 1', 'callback 1',
                'process 4', 'callback 4',
                'process 3', 'callback 3'
            ]);
            expect(q.concurrency).to.equal(2);
            expect(q.length()).to.equal(0);
            done();
        };
    });

    it('default concurrency', function(done) {
        var call_order = [],
            delays = [40,20,60,20];

        // order of completion: 1,2,3,4

        var q = async.queue(function (task, callback) {
            setTimeout(function () {
                call_order.push('process ' + task);
                callback('error', 'arg');
            }, delays.shift());
        });

        q.push(1, function (err, arg) {
            expect(err).to.equal('error');
            expect(arg).to.equal('arg');
            expect(q.length()).to.equal(3);
            call_order.push('callback ' + 1);
        });
        q.push(2, function (err, arg) {
            expect(err).to.equal('error');
            expect(arg).to.equal('arg');
            expect(q.length()).to.equal(2);
            call_order.push('callback ' + 2);
        });
        q.push(3, function (err, arg) {
            expect(err).to.equal('error');
            expect(arg).to.equal('arg');
            expect(q.length()).to.equal(1);
            call_order.push('callback ' + 3);
        });
        q.push(4, function (err, arg) {
            expect(err).to.equal('error');
            expect(arg).to.equal('arg');
            expect(q.length()).to.equal(0);
            call_order.push('callback ' + 4);
        });
        expect(q.length()).to.equal(4);
        expect(q.concurrency).to.equal(1);

        q.drain = function () {
            expect(call_order).to.eql([
                'process 1', 'callback 1',
                'process 2', 'callback 2',
                'process 3', 'callback 3',
                'process 4', 'callback 4'
            ]);
            expect(q.concurrency).to.equal(1);
            expect(q.length()).to.equal(0);
            done();
        };
    });

    it('zero concurrency', function(done){
        expect(function () {
            async.queue(function (task, callback) {
                callback(null, task);
            }, 0);
        }).to.throw();
        done();
    });

    it('error propagation', function(done){
        var results = [];

        var q = async.queue(function (task, callback) {
            callback(task.name === 'foo' ? new Error('fooError') : null);
        }, 2);

        q.drain = function() {
            expect(results).to.eql(['bar', 'fooError']);
            done();
        };

        q.push({name: 'bar'}, function (err) {
            if(err) {
                results.push('barError');
                return;
            }

            results.push('bar');
        });

        q.push({name: 'foo'}, function (err) {
            if(err) {
                results.push('fooError');
                return;
            }

            results.push('foo');
        });
    });

    it('global error handler', function(done){
        var results = [];

        var q = async.queue(function (task, callback) {
            callback(task.name === 'foo' ? new Error('fooError') : null);
        }, 2);

        q.error = function(error, task) {
            expect(error).to.exist;
            expect(error.message).to.equal('fooError');
            expect(task.name).to.equal('foo');
            results.push('fooError');
        };

        q.drain = function() {
            expect(results).to.eql(['fooError', 'bar']);
            done();
        };

        q.push({name: 'foo'});

        q.push({name: 'bar'}, function(error) {
            expect(error).to.not.exist;
            results.push('bar');
        });
    });

    // The original queue implementation allowed the concurrency to be changed only
    // on the same event loop during which a task was added to the queue. This
    // test attempts to be a more robust test.
    // Start with a concurrency of 1. Wait until a leter event loop and change
    // the concurrency to 2. Wait again for a later loop then verify the concurrency
    // Repeat that one more time by chaning the concurrency to 5.
    it('changing concurrency', function(done) {

        var q = async.queue(function(task, callback){
            setTimeout(function(){
                callback();
            }, 10);
        }, 1);

        for(var i = 0; i < 50; i++){
            q.push('');
        }

        q.drain = function(){
            done();
        };

        setTimeout(function(){
            expect(q.concurrency).to.equal(1);
            q.concurrency = 2;
            setTimeout(function(){
                expect(q.running()).to.equal(2);
                q.concurrency = 5;
                setTimeout(function(){
                    expect(q.running()).to.equal(5);
                }, 40);
            }, 40);
        }, 40);
    });

    it('push without callback', function(done) {
        var call_order = [],
            delays = [40,20,60,20];

        // worker1: --1-4
        // worker2: -2---3
        // order of completion: 2,1,4,3

        var q = async.queue(function (task, callback) {
            setTimeout(function () {
                call_order.push('process ' + task);
                callback('error', 'arg');
            }, delays.shift());
        }, 2);

        q.push(1);
        q.push(2);
        q.push(3);
        q.push(4);

        q.drain = function () {
            expect(call_order).to.eql([
                'process 2',
                'process 1',
                'process 4',
                'process 3'
            ]);
            done();
        };
    });

    it('push with non-function', function(done) {
        var q = async.queue(function () {}, 1);
        expect(function () {
            q.push({}, 1);
        }).to.throw();
        done();
    });

    it('unshift', function(done) {
        var queue_order = [];

        var q = async.queue(function (task, callback) {
            queue_order.push(task);
            callback();
        }, 1);

        q.unshift(4);
        q.unshift(3);
        q.unshift(2);
        q.unshift(1);

        setTimeout(function () {
            expect(queue_order).to.eql([ 1, 2, 3, 4 ]);
            done();
        }, 100);
    });

    it('too many callbacks', function(done) {
        var q = async.queue(function (task, callback) {
            callback();
            expect(function() {
                callback();
            }).to.throw();
            done();
        }, 2);

        q.push(1);
    });

    it('bulk task', function(done) {
        var call_order = [],
            delays = [40,20,60,20];

        // worker1: --1-4
        // worker2: -2---3
        // order of completion: 2,1,4,3

        var q = async.queue(function (task, callback) {
            setTimeout(function () {
                call_order.push('process ' + task);
                callback('error', task);
            }, delays.splice(0,1)[0]);
        }, 2);

        q.push( [1,2,3,4], function (err, arg) {
            expect(err).to.equal('error');
            call_order.push('callback ' + arg);
        });

        expect(q.length()).to.equal(4);
        expect(q.concurrency).to.equal(2);

        q.drain = function () {
            expect(call_order).to.eql([
                'process 2', 'callback 2',
                'process 1', 'callback 1',
                'process 4', 'callback 4',
                'process 3', 'callback 3'
            ]);
            expect(q.concurrency).to.equal(2);
            expect(q.length()).to.equal(0);
            done();
        };
    });

    it('idle', function(done) {
        var q = async.queue(function (task, callback) {
            // Queue is busy when workers are running
            expect(q.idle()).to.equal(false);
            callback();
        }, 1);

        // Queue is idle before anything added
        expect(q.idle()).to.equal(true);

        q.unshift(4);
        q.unshift(3);
        q.unshift(2);
        q.unshift(1);

        // Queue is busy when tasks added
        expect(q.idle()).to.equal(false);

        q.drain = function() {
            // Queue is idle after drain
            expect(q.idle()).to.equal(true);
            done();
        };
    });

    it('pause', function(done) {
        var call_order = [],
            task_timeout = 80,
            pause_timeout = task_timeout * 2.5,
            resume_timeout = task_timeout * 4.5,
            tasks = [ 1, 2, 3, 4, 5, 6 ],

            elapsed = (function () {
                var start = Date.now();
                return function () {
                    return Math.round((Date.now() - start) / task_timeout) * task_timeout;
                };
            })();

        var q = async.queue(function (task, callback) {
            call_order.push('process ' + task);
            call_order.push('timeout ' + elapsed());
            callback();
        });

        function pushTask () {
            var task = tasks.shift();
            if (!task) { return; }
            setTimeout(function () {
                q.push(task);
                pushTask();
            }, task_timeout);
        }
        pushTask();

        setTimeout(function () {
            q.pause();
            expect(q.paused).to.equal(true);
        }, pause_timeout);

        setTimeout(function () {
            q.resume();
            expect(q.paused).to.equal(false);
        }, resume_timeout);

        setTimeout(function () {
            expect(call_order).to.eql([
                'process 1', 'timeout ' + task_timeout,
                'process 2', 'timeout ' + task_timeout * 2,
                'process 3', 'timeout ' + task_timeout * 5,
                'process 4', 'timeout ' + task_timeout * 5,
                'process 5', 'timeout ' + task_timeout * 5,
                'process 6', 'timeout ' + task_timeout * 6
            ]);
            done();
        }, (task_timeout * tasks.length) + pause_timeout + resume_timeout);
    });

    it('pause in worker with concurrency', function(done) {
            var call_order = [];
            var q = async.queue(function (task, callback) {
                if (task.isLongRunning) {
                    q.pause();
                    setTimeout(function () {
                        call_order.push(task.id);
                        q.resume();
                        callback();
                    }, 50);
                }
                else {
                    call_order.push(task.id);
                    setTimeout(callback, 10);
                }
            }, 10);

            q.push({ id: 1, isLongRunning: true});
            q.push({ id: 2 });
            q.push({ id: 3 });
            q.push({ id: 4 });
            q.push({ id: 5 });

            q.drain = function () {
                expect(call_order).to.eql([1, 2, 3, 4, 5]);
                done();
            };
        });

    it('pause with concurrency', function(done) {
        var call_order = [],
            task_timeout = 40,
            pause_timeout = task_timeout / 2,
            resume_timeout = task_timeout * 2.75,
            tasks = [ 1, 2, 3, 4, 5, 6 ],

            elapsed = (function () {
                var start = Date.now();
                return function () {
                    return Math.round((Date.now() - start) / task_timeout) * task_timeout;
                };
            })();

        var q = async.queue(function (task, callback) {
            setTimeout(function () {
                call_order.push('process ' + task);
                call_order.push('timeout ' + elapsed());
                callback();
            }, task_timeout);
        }, 2);

        q.push(tasks);

        setTimeout(function () {
            q.pause();
            expect(q.paused).to.equal(true);
        }, pause_timeout);

        setTimeout(function () {
            q.resume();
            expect(q.paused).to.equal(false);
        }, resume_timeout);

        setTimeout(function () {
            expect(q.running()).to.equal(2);
        }, resume_timeout + 10);

        setTimeout(function () {
            expect(call_order).to.eql([
                'process 1', 'timeout ' + task_timeout,
                'process 2', 'timeout ' + task_timeout,
                'process 3', 'timeout ' + task_timeout * 4,
                'process 4', 'timeout ' + task_timeout * 4,
                'process 5', 'timeout ' + task_timeout * 5,
                'process 6', 'timeout ' + task_timeout * 5
            ]);
            done();
        }, (task_timeout * tasks.length) + pause_timeout + resume_timeout);
    });

    it('start paused', function(done) {
        var q = async.queue(function (task, callback) {
            setTimeout(function () {
                callback();
            }, 40);
        }, 2);
        q.pause();

        q.push([1, 2, 3]);

        setTimeout(function () {
            q.resume();
        }, 5);

        setTimeout(function () {
            expect(q.tasks.length).to.equal(1);
            expect(q.running()).to.equal(2);
            q.resume();
        }, 15);

        q.drain = function () {
            done();
        };
    });

    it('kill', function(done) {
        var q = async.queue(function (task, callback) {
            setTimeout(function () {
                throw new Error("Function should never be called");
            }, 20);
        }, 1);
        q.drain = function() {
            throw new Error("Function should never be called");
        };

        q.push(0);

        q.kill();

        setTimeout(function() {
            expect(q.length()).to.equal(0);
            done();
        }, 40);
    });

    it('events', function(done) {
        var calls = [];
        var q = async.queue(function(task, cb) {
            // nop
            calls.push('process ' + task);
            setTimeout(cb, 10);
        }, 3);
        q.concurrency = 3;

        q.saturated = function() {
            assert(q.running() == 3, 'queue should be saturated now');
            calls.push('saturated');
        };
        q.empty = function() {
            assert(q.length() === 0, 'queue should be empty now');
            calls.push('empty');
        };
        q.drain = function() {
            assert(
                q.length() === 0 && q.running() === 0,
                'queue should be empty now and no more workers should be running'
            );
            calls.push('drain');
            expect(calls).to.eql([
                'process foo',
                'process bar',
                'saturated',
                'process zoo',
                'foo cb',
                'saturated',
                'process poo',
                'bar cb',
                'empty',
                'saturated',
                'process moo',
                'zoo cb',
                'poo cb',
                'moo cb',
                'drain'
            ]);
            done();
        };
        q.push('foo', function () {calls.push('foo cb');});
        q.push('bar', function () {calls.push('bar cb');});
        q.push('zoo', function () {calls.push('zoo cb');});
        q.push('poo', function () {calls.push('poo cb');});
        q.push('moo', function () {calls.push('moo cb');});
    });

    it('empty', function(done) {
        var calls = [];
        var q = async.queue(function(task, cb) {
            // nop
            calls.push('process ' + task);
            async.setImmediate(cb);
        }, 3);

        q.drain = function() {
            assert(
                q.length() === 0 && q.running() === 0,
                'queue should be empty now and no more workers should be running'
            );
            calls.push('drain');
            expect(calls).to.eql([
                'drain'
            ]);
            done();
        };
        q.push([]);
    });

    it('saturated', function(done) {
        var saturatedCalled = false;
        var q = async.queue(function(task, cb) {
            async.setImmediate(cb);
        }, 2);

        q.saturated = function () {
            saturatedCalled = true;
        };
        q.drain = function () {
            assert(saturatedCalled, "saturated not called");
            done();
        };

        q.push(['foo', 'bar', 'baz', 'moo']);
    });

    it('started', function(done) {

        var q = async.queue(function(task, cb) {
            cb(null, task);
        });

        expect(q.started).to.equal(false);
        q.push([]);
        expect(q.started).to.equal(true);
        done();
    });

    context('q.saturated(): ', function() {
        it('should call the saturated callback if tasks length is concurrency', function(done) {
            var calls = [];
            var q = async.queue(function(task, cb) {
                calls.push('process ' + task);
                async.setImmediate(cb);
            }, 4);
            q.saturated = function() {
                calls.push('saturated');
            };
            q.empty = function() {
                expect(calls.indexOf('saturated')).to.be.above(-1);
                setTimeout(function() {
                    expect(calls).eql([
                        'process foo0',
                        'process foo1',
                        'process foo2',
                        "saturated",
                        'process foo3',
                        'foo0 cb',
                        "saturated",
                        'process foo4',
                        'foo1 cb',
                        'foo2 cb',
                        'foo3 cb',
                        'foo4 cb'
                    ]);
                    done();
                }, 50);
            };
            q.push('foo0', function () {calls.push('foo0 cb');});
            q.push('foo1', function () {calls.push('foo1 cb');});
            q.push('foo2', function () {calls.push('foo2 cb');});
            q.push('foo3', function () {calls.push('foo3 cb');});
            q.push('foo4', function () {calls.push('foo4 cb');});
        });
    });

    context('q.unsaturated(): ',function() {
        it('should have a default buffer property that equals 25% of the concurrenct rate', function(done){
            var calls = [];
            var q = async.queue(function(task, cb) {
                // nop
                calls.push('process ' + task);
                async.setImmediate(cb);
            }, 10);
            expect(q.buffer).to.equal(2.5);
            done();
        });
        it('should allow a user to change the buffer property', function(done){
            var calls = [];
            var q = async.queue(function(task, cb) {
                // nop
                calls.push('process ' + task);
                async.setImmediate(cb);
            }, 10);
            q.buffer = 4;
            expect(q.buffer).to.not.equal(2.5);
            expect(q.buffer).to.equal(4);
            done();
        });
        it('should call the unsaturated callback if tasks length is less than concurrency minus buffer', function(done){
            var calls = [];
            var q = async.queue(function(task, cb) {
                calls.push('process ' + task);
                async.setImmediate(cb);
            }, 4);
            q.unsaturated = function() {
                calls.push('unsaturated');
            };
            q.empty = function() {
                expect(calls.indexOf('unsaturated')).to.be.above(-1);
                setTimeout(function() {
                    expect(calls).eql([
                        'process foo0',
                        'process foo1',
                        'process foo2',
                        'process foo3',
                        'foo0 cb',
                        'unsaturated',
                        'process foo4',
                        'foo1 cb',
                        'unsaturated',
                        'foo2 cb',
                        'unsaturated',
                        'foo3 cb',
                        'unsaturated',
                        'foo4 cb',
                        'unsaturated'
                    ]);
                    done();
                }, 50);
            };
            q.push('foo0', function () {calls.push('foo0 cb');});
            q.push('foo1', function () {calls.push('foo1 cb');});
            q.push('foo2', function () {calls.push('foo2 cb');});
            q.push('foo3', function () {calls.push('foo3 cb');});
            q.push('foo4', function () {calls.push('foo4 cb');});
        });
    });
});


text:
require 'spec_helper.rb'
require 's3sync/cli'
require 's3sync/config'
require 's3sync/sync'

include S3Sync

describe "Parsing command line arguments" do

  describe "Processing the final destination based on how the user expressed the source" do

    it "Put the local etc directory itself into S3" do
      source = "/etc"
      destination = "mybucket:pre"

      # This will yield S3 keys named  pre/etc/...
      expect(SyncCommand.process_destination(source, destination)).to eql(["/etc", ["pre/etc/", "mybucket"]])
    end

    it "Put the contents of the local /etc dir into S3, rename dir" do
      source = "/etc/"
      destination = "mybucket:pre/etcbackup"

      # This will yield S3 keys named  pre/etcbackup/...
      expect(SyncCommand.process_destination(source, destination)).to eql(["/etc/", ["pre/etcbackup/", "mybucket"]])
    end

    it "Put the contents of the local /tmp/sync/ in to the root of an S3 bucket" do
      source = "/tmp/sync/"
      destination = "mybucket:"

      # This will yield S3 keys named  '...' (The root is just empty)
      expect(SyncCommand.process_destination(source, destination)).to eql(["/tmp/sync/", ["", "mybucket"]])
    end

    it "Put contents of S3 \"directory\" etc into local dir" do
      source = "mybucket:pre/etc/"
      destination = "/root/etcrestore"

      # This will yield local files at  /root/etcrestore/...
      expect(SyncCommand.process_destination(source, destination)).to eql([["pre/etc/", "mybucket"], "/root/etcrestore/"])
    end

    it "Put the contents of S3 \"directory\" etc into a local dir named etc" do
      source = "mybucket:pre/etc"
      destination = "/root"

      # This will yield local files at  /root/etc/...
      expect(SyncCommand.process_destination(source, destination)).to eql([["pre/etc", "mybucket"], "/root/etc/"])
    end

    it "Put S3 nodes under the key pre/etc/ to the local dir etcrestore" do
      source = "mybucket:pre/etc/"
      destination = "/root/etcrestore"

      # This will yield local files at  /root/etcrestore/...
      expect(SyncCommand.process_destination(source, destination)).to eql([["pre/etc/", "mybucket"], "/root/etcrestore/"])
    end

    it "Put S3 nodes under an empty key (root) to the local dir /tmp/lib" do
      source = "mybucket:"
      destination = "/tmp/lib"

      # This will yield local files at  /root/etcrestore/...
      expect(SyncCommand.process_destination(source, destination)).to eql([["", "mybucket"], "/tmp/lib/"])
    end
  end

  it "Should calculate the right destination for each path" do
    file = "pre/etc/sub/path/blah.txt"   # This is how it comes from s3
    source = "mybucket:pre/etc/"
    destination = "/root/etcrestore"

    expect(SyncCommand.process_file_destination(source, destination, file)).to eql("/root/etcrestore/sub/path/blah.txt")
  end

  it "Put S3 files under an empty key (root) to the local dir /tmp/lib" do
    source = "mybucket:"
    destination = "/tmp/lib"
    file = "myfile.rb"

    # This will yield local files at  /tmp/lib/...
    expect(SyncCommand.process_file_destination(source, destination, file)).to eql("/tmp/lib/myfile.rb")
  end

  it "Returning locations based on the parsed destination" do
    source = "/etc"
    destination = "mybucket:pre"

    # When I parse the above arguments using the SyncCommand
    src_location, dst_location = SyncCommand.parse_params [source, destination]

    # Then I see I got the locations with the right params
    expect(src_location).to eql(S3Sync::Location.new("/etc"))
    expect(dst_location).to eql(S3Sync::Location.new("pre/etc/", "mybucket"))
  end

  it "Location should be parsed when it is remote with no path" do
    source = "/etc"
    destination = "mybucket:"

    # When I parse the above arguments using the SyncCommand
    src_location, dst_location = SyncCommand.parse_params [source, destination]

    # Then I see I got the locations with the right params
    expect(src_location).to eql(S3Sync::Location.new("/etc"))
    expect(dst_location).to eql(S3Sync::Location.new("etc/", "mybucket"))
  end

  it "should be possible to detect if a location is remote" do
    expect(SyncCommand.remote_prefix?("bucket:prefix")).to be(true)
    expect(SyncCommand.remote_prefix?("path")).to be(false)
    expect(SyncCommand.remote_prefix?("C://blah")).to be(false)  # We support windows, LOL
  end
end

describe "Comparing file lists" do

  it "should be possible to describe nodes with their paths and size" do

    # Full test
    node = Node.new "path/to", "file1", 10
    expect(node.path).to eql("file1")
    expect(node.full).to eql("path/to/file1")
    expect(node.size).to eql(10)

    # Alternative constructor scenarios
    node = Node.new "", "file1", 10
    expect(node.path).to eql("file1")
  end

  it "should be possible to compare two lists of files" do

    # Given that I have two lists of Nodes to compare
    hash1 = {
      "file1" => Node.new("", "file1", 10),
      "file2" => Node.new("", "file2", 12),
      "file3" => Node.new("", "file3", 12),
    }

    hash2 = {
      "file1" => Node.new("", "file1", 10),
      "file2" => Node.new("", "file2", 22),
      "file4" => Node.new("", "file4", 22),
    }

    # When I compare those two file lists
    same_in_both, to_be_added_to_list2, to_be_removed_from_list2 = SyncCommand.cmp hash1, hash2

    # Then I see that the three lists that I requested were returned with the
    # right content
    expect(same_in_both).to eq([Node.new("", "file1", 10)])  # Just testing our == operator
    expect(same_in_both).to eql([Node.new("", "file1", 10)])
    expect(to_be_added_to_list2).to eql([Node.new("", "file2", 12), Node.new("", "file3", 12)])
    expect(to_be_removed_from_list2).to eql([Node.new("", "file4", 22)])
  end

  it 'can compare small files with an extra comparator' do
    large_file_size = S3Sync::Node::SMALL_FILE + 1024
    # Given that I have two lists of Nodes to compare
    hash1 = {
      "file1" => Node.new("", "file1", 10, -> { 'same' }),
      "file2" => Node.new("", "file2", 22, -> { 'abc' }),
      "file3" => Node.new("", "file3", 12, -> { fail }),
      "file5" => Node.new("", "file5", large_file_size, -> { 'abc' }),
    }

    hash2 = {
      "file1" => Node.new("", "file1", 10, -> { 'same' }),
      "file2" => Node.new("", "file2", 22, -> { 'def' }),
      "file4" => Node.new("", "file4", 22, -> { fail }),
      "file5" => Node.new("", "file5", large_file_size, -> { 'def' }),
    }

    # When I compare those two file lists
    same_in_both, to_be_added_to_list2, to_be_removed_from_list2 = SyncCommand.cmp hash1, hash2

    # Then I see that the three lists that I requested were returned with the
    # right content
    expect(same_in_both).to eq([Node.new("", "file1", 10), Node.new('', 'file5', large_file_size)])
    expect(to_be_added_to_list2).to eql([Node.new("", "file2", 22), Node.new("", "file3", 12)])
    expect(to_be_removed_from_list2).to eql([Node.new("", "file4", 22)])
  end
end

redpajama_set_name: RedPajamaGithub


text:
#include "helper.h"
#include "initLB.h"
#include "LBDefinitions.h"

/**
 * Reads the parameters for the lid driven cavity scenario from a config file.
 * Throws an error if number of program arguments does not equal 2.
 **/
int readParameters(
    int *length,                        /* reads domain size. Parameter name: "xlength" */
    double *tau,                        /* relaxation parameter tau. Parameter name: "tau" */
    double *velocity,               /* velocity of the lid. Parameter name: "characteristicvelocity" */
    int *timesteps,                     /* number of timesteps. Parameter name: "timesteps" */
    int *timestepsPerPlotting,          /* timesteps between subsequent VTK plots. Parameter name: "vtkoutput" */
    int argc,                           /* number of arguments. Should equal 2 (program + name of config file */
    char *argv[],                       /* argv[1] shall contain the path to the config file */
    char *problem,                      /* specifies the considered problem scenario (parabolic or constant inflow )*/
    double *ro_ref,                     /* reference density nomally set to 1 */
    double *ro_in,                      /* density of inflow/outflow */
    int *boundaries                     /* definition of the type of boundaries on each one of the walls, for definitions see LBDefinitios.h*/
    ){

    if (argc != 2) {
        ERROR("number of arguments is incorrect");
    }
    char filename[20] ;
    strcpy(filename,argv[1]);
    const char *szFileName = strcat( filename,".dat"); /* Concatenate .dat so the imput is independent of extesion*/

    read_int(szFileName, "zlength", &length[0]);
    read_int(szFileName, "ylength", &length[1]);
    read_int(szFileName, "xlength", &length[2]);
    read_double(szFileName,"tau", tau);

    read_double(szFileName, "velocity_x", &velocity[0]);
    read_double(szFileName, "velocity_y", &velocity[1]);
    read_double(szFileName, "velocity_z", &velocity[2]);

    read_int(szFileName, "timesteps", timesteps);
    read_int(szFileName, "vtkoutput", timestepsPerPlotting);
    read_string(szFileName, "problem", problem);
    read_double(szFileName,"ro_ref", ro_ref);
    read_double(szFileName,"ro_in", ro_in);

    read_int(szFileName, "wall_x0", &boundaries[0]);
    read_int(szFileName, "wall_xmax", &boundaries[1]);
    read_int(szFileName, "wall_y0", &boundaries[2]);
    read_int(szFileName, "wall_ymax", &boundaries[3]);
    read_int(szFileName, "wall_z0", &boundaries[4]);
    read_int(szFileName, "wall_zmax", &boundaries[5]);

    if (strcmp(problem, PARABOLIC_SCENARIO) != 0 && strcmp(problem, CONSTANT_SCENARIO) != 0) {
        ERROR("Unrecognized scenario");
    }

    return 0;
}

void initialiseCell(double *collideField, double *streamField, int *flagField, int *length, int x, int y, int z, int flag) {
    int i;

    int n[3] = { length[0] + 2, length[1] + 2, length[2] + 2 };

    *getFlag(flagField, x, y, z, n) = flag;    /*asigns the flag to the specified cell */

    for (i = 0; i < Q; ++i){
        *getEl(streamField, x, y, z, i, n) = LATTICEWEIGHTS[i];    /*Insert on each cell the initial value */
        *getEl(collideField, x, y, z, i, n) = LATTICEWEIGHTS[i];
    }
}

/**
 * Checks that in input image there was no too thin boundaries
 */
void checkForbiddenPatterns(int ** image, int * length) {
    int x, z;
    int c1, c2, c3, c4, result;

    for (x = 0; x <= length[2]; x++) {
        for (z = 0; z <= length[0]; z++) {
            c1 = image[x][z] == FLUID;
            c2 = image[x + 1][z] == FLUID;
            c3 = image[x][z + 1] == FLUID;
            c4 = image[x + 1][z + 1] == FLUID;
            result = c1 << 3 | c2 << 2 | c3 << 1 | c4;

            if (result == 6 || result == 9) {  // result == 0b0110 or 0b1001
                ERROR("forbidden boundary");
            }
        }
    }
}

void initialiseFields(double *collideField, double *streamField, int *flagField, int * length, int * boundaries, char *argv[]){
    int x, y, z;
    int ** image;
    char filename[20] ;

    strcpy(filename,argv[1]);                   /*Copy the value from argv to filename to not modify the original one*/
    strcat( filename,".pgm");
    image = read_pgm(filename);/* Concatenate .pgm so the imput is independent of extension*/
    checkForbiddenPatterns(image, length);

    for (z = 0; z <= length[0]; z++) {
        for (x = 0; x <= length[2]; x++) {
            printf("%d ", image[x][z]);
        }
        printf("\n");
    }

    /*
    *   Definition of the fields
    *   The structure of the looping makes possible to define the boudaries
    */
    z = 0;
    for (y = 0; y <= length[1]+1; ++y){
        for (x = 0; x <= length[2]+1; ++x){
            initialiseCell(collideField, streamField, flagField, length, x, y, z, boundaries[4]);   /* Loop for the down wall of the cavity*/
        }
    }

    for (z = 1; z <= length[0]; ++z){
        y = 0;
        for (x = 0; x <= length[2]+1; ++x){
            initialiseCell(collideField, streamField, flagField, length, x, y, z, boundaries[2]);   /* Loop for the front wall of the cavity*/
        }

        for (y = 1; y <= length[1]; ++y){
            x = 0;
            initialiseCell(collideField, streamField, flagField, length, x, y, z, boundaries[0]);   /* Loop for the left wall of the cavity*/
            for (x = 1; x <= length[2]; ++x){
                initialiseCell(collideField, streamField, flagField, length, x, y, z, image[x][z]);           /* Loop for the interior points*/
            }
            x = length[2]+1;
            initialiseCell(collideField, streamField, flagField, length, x, y, z, boundaries[1]);   /* Loop for the right wall of the cavity*/
        }

        y = length[1]+1;
        for (x = 0; x <= length[2]+1; ++x){
            initialiseCell(collideField, streamField, flagField, length, x, y, z, boundaries[3]);   /* Loop for the back wall of the cavity*/
        }
    }

    z = length[0]+1;
    for (y = 0; y <= length[1]+1; ++y){
        for (x = 0; x < length[2]+2; ++x){
            initialiseCell(collideField, streamField, flagField, length, x, y, z, boundaries[5]);   /* Loop for the up wall of the cavity*/
        }
    }
    free_imatrix( image, 0, length[0]+2,0,length[2]+2);
}


text:
//********************************************************************
//*
//* Warning: This file was generated by MDE4CPP Generator
//*
//********************************************************************

#ifndef STANDARDPROFILE_FRAMEWORK__HPP
#define STANDARDPROFILE_FRAMEWORK__HPP

//Forward Declaration for used types
namespace uml
{
	class Package;
}

// base class includes
#include "uml/Stereotype.hpp"

namespace StandardProfile
{
	class Framework : virtual public uml::Stereotype
	{
		protected:
			Framework(){}

		public:
			//destructor
			virtual ~Framework() {}

			//*********************************
			// Getter Setter
			//*********************************
			virtual std::weak_ptr<uml::Package> getBase_Package() const  = 0;
			virtual void setBase_Package(std::weak_ptr<uml::Package> _base_Package) = 0;

			//*********************************
			// Operations
			//*********************************

		protected:
			//*********************************
			// Members
			//*********************************
			std::weak_ptr<uml::Package> m_base_Package;

		public:
			//*********************************
			// Union Getter
			//*********************************

	};
}
#endif /* end of include guard: STANDARDPROFILE_FRAMEWORK__HPP */


text:
"""
Tests for L{twisted.web._auth}.
"""

import md5, sha

from zope.interface import implements
from zope.interface.verify import verifyObject

from twisted.trial import unittest
from twisted.cred import error, portal
from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse
from twisted.cred.credentials import IUsernamePassword
from twisted.internet.address import IPv4Address
from twisted.web.iweb import ICredentialFactory, IUsernameDigestHash
from twisted.web.resource import IResource, Resource
from twisted.web._auth import basic, digest
from twisted.web._auth.wrapper import HTTPAuthSessionWrapper, UnauthorizedResource
from twisted.web._auth.basic import BasicCredentialFactory
from twisted.web._auth.digest import calcHA1, calcHA2, calcResponse
from twisted.web.server import NOT_DONE_YET
from twisted.web.static import Data

from twisted.web.test.test_web import DummyRequest

def b64encode(s):
    return s.encode('base64').strip()

class FakeDigestCredentialFactory(digest.DigestCredentialFactory):
    """
    A Fake Digest Credential Factory that generates a predictable
    nonce and opaque
    """
    def __init__(self, *args, **kwargs):
        super(FakeDigestCredentialFactory, self).__init__(*args, **kwargs)
        self.privateKey = "0"

    def generateNonce(self):
        """
        Generate a static nonce
        """
        return '178288758716122392881254770685'

    def _getTime(self):
        """
        Return a stable time
        """
        return 0

class BasicAuthTestsMixin:
    """
    L{TestCase} mixin class which defines a number of tests for
    L{basic.BasicCredentialFactory}.  Because this mixin defines C{setUp}, it
    must be inherited before L{TestCase}.
    """
    def setUp(self):
        self.request = self.makeRequest()
        self.realm = 'foo'
        self.username = 'dreid'
        self.password = 'S3CuR1Ty'
        self.credentialFactory = basic.BasicCredentialFactory(self.realm)

    def makeRequest(self, method='GET', clientAddress=None):
        """
        Create a request object to be passed to
        L{basic.BasicCredentialFactory.decode} along with a response value.
        Override this in a subclass.
        """
        raise NotImplementedError("%r did not implement makeRequest" % (
                self.__class__,))

    def test_interface(self):
        """
        L{BasicCredentialFactory} implements L{ICredentialFactory}.
        """
        self.assertTrue(
            verifyObject(ICredentialFactory, self.credentialFactory))

    def test_usernamePassword(self):
        """
        L{basic.BasicCredentialFactory.decode} turns a base64-encoded response
        into a L{UsernamePassword} object with a password which reflects the
        one which was encoded in the response.
        """
        response = b64encode('%s:%s' % (self.username, self.password))

        creds = self.credentialFactory.decode(response, self.request)
        self.assertTrue(IUsernamePassword.providedBy(creds))
        self.assertTrue(creds.checkPassword(self.password))
        self.assertFalse(creds.checkPassword(self.password + 'wrong'))

    def test_incorrectPadding(self):
        """
        L{basic.BasicCredentialFactory.decode} decodes a base64-encoded
        response with incorrect padding.
        """
        response = b64encode('%s:%s' % (self.username, self.password))
        response = response.strip('=')

        creds = self.credentialFactory.decode(response, self.request)
        self.assertTrue(verifyObject(IUsernamePassword, creds))
        self.assertTrue(creds.checkPassword(self.password))

    def test_invalidEncoding(self):
        """
        L{basic.BasicCredentialFactory.decode} raises L{LoginFailed} if passed
        a response which is not base64-encoded.
        """
        response = 'x' # one byte cannot be valid base64 text
        self.assertRaises(
            error.LoginFailed,
            self.credentialFactory.decode, response, self.makeRequest())

    def test_invalidCredentials(self):
        """
        L{basic.BasicCredentialFactory.decode} raises L{LoginFailed} when
        passed a response which is not valid base64-encoded text.
        """
        response = b64encode('123abc+/')
        self.assertRaises(
            error.LoginFailed,
            self.credentialFactory.decode,
            response, self.makeRequest())

class RequestMixin:
    def makeRequest(self, method='GET', clientAddress=None):
        """
        Create a L{DummyRequest} (change me to create a
        L{twisted.web.http.Request} instead).
        """
        request = DummyRequest('/')
        request.method = method
        request.client = clientAddress
        return request

class BasicAuthTestCase(RequestMixin, BasicAuthTestsMixin, unittest.TestCase):
    """
    Basic authentication tests which use L{twisted.web.http.Request}.
    """

class DigestAuthTestsMixin:
    """
    L{TestCase} mixin class which defines a number of tests for
    L{digest.DigestCredentialFactory}.  Because this mixin defines C{setUp}, it
    must be inherited before L{TestCase}.
    """
    def setUp(self):
        """
        Create a DigestCredentialFactory for testing
        """
        self.username = "foobar"
        self.password = "bazquux"
        self.realm = "test realm"
        self.algorithm = "md5"
        self.cnonce = "29fc54aa1641c6fa0e151419361c8f23"
        self.qop = "auth"
        self.uri = "/write/"
        self.clientAddress = IPv4Address('TCP', '10.2.3.4', 43125)
        self.request = self.makeRequest('GET', self.clientAddress)
        self.credentialFactory = digest.DigestCredentialFactory(
            self.algorithm, self.realm)

    def test_MD5HashA1(self, _algorithm='md5', _hash=md5.md5):
        """
        L{calcHA1} accepts the C{'md5'} algorithm and returns an MD5 hash of
        its parameters, excluding the nonce and cnonce.
        """
        nonce = 'abc123xyz'
        hashA1 = calcHA1(_algorithm, self.username, self.realm, self.password,
                         nonce, self.cnonce)
        a1 = '%s:%s:%s' % (self.username, self.realm, self.password)
        expected = _hash(a1).hexdigest()
        self.assertEqual(hashA1, expected)

    def test_MD5SessionHashA1(self):
        """
        L{calcHA1} accepts the C{'md5-sess'} algorithm and returns an MD5 hash
        of its parameters, including the nonce and cnonce.
        """
        nonce = 'xyz321abc'
        hashA1 = calcHA1('md5-sess', self.username, self.realm, self.password,
                         nonce, self.cnonce)
        a1 = '%s:%s:%s' % (self.username, self.realm, self.password)
        ha1 = md5.md5(a1).digest()
        a1 = '%s:%s:%s' % (ha1, nonce, self.cnonce)
        expected = md5.md5(a1).hexdigest()
        self.assertEqual(hashA1, expected)

    def test_SHAHashA1(self):
        """
        L{calcHA1} accepts the C{'sha'} algorithm and returns a SHA hash of its
        parameters, excluding the nonce and cnonce.
        """
        self.test_MD5HashA1('sha', sha.sha)

    def test_MD5HashA2Auth(self, _algorithm='md5', _hash=md5.md5):
        """
        L{calcHA2} accepts the C{'md5'} algorithm and returns an MD5 hash of
        its arguments, excluding the entity hash for QOP other than
        C{'auth-int'}.
        """
        method = 'GET'
        hashA2 = calcHA2(_algorithm, method, self.uri, 'auth', None)
        a2 = '%s:%s' % (method, self.uri)
        expected = _hash(a2).hexdigest()
        self.assertEqual(hashA2, expected)

    def test_MD5HashA2AuthInt(self, _algorithm='md5', _hash=md5.md5):
        """
        L{calcHA2} accepts the C{'md5'} algorithm and returns an MD5 hash of
        its arguments, including the entity hash for QOP of C{'auth-int'}.
        """
        method = 'GET'
        hentity = 'foobarbaz'
        hashA2 = calcHA2(_algorithm, method, self.uri, 'auth-int', hentity)
        a2 = '%s:%s:%s' % (method, self.uri, hentity)
        expected = _hash(a2).hexdigest()
        self.assertEqual(hashA2, expected)

    def test_MD5SessHashA2Auth(self):
        """
        L{calcHA2} accepts the C{'md5-sess'} algorithm and QOP of C{'auth'} and
        returns the same value as it does for the C{'md5'} algorithm.
        """
        self.test_MD5HashA2Auth('md5-sess')

    def test_MD5SessHashA2AuthInt(self):
        """
        L{calcHA2} accepts the C{'md5-sess'} algorithm and QOP of C{'auth-int'}
        and returns the same value as it does for the C{'md5'} algorithm.
        """
        self.test_MD5HashA2AuthInt('md5-sess')

    def test_SHAHashA2Auth(self):
        """
        L{calcHA2} accepts the C{'sha'} algorithm and returns a SHA hash of
        its arguments, excluding the entity hash for QOP other than
        C{'auth-int'}.
        """
        self.test_MD5HashA2Auth('sha', sha.sha)

    def test_SHAHashA2AuthInt(self):
        """
        L{calcHA2} accepts the C{'sha'} algorithm and returns a SHA hash of
        its arguments, including the entity hash for QOP of C{'auth-int'}.
        """
        self.test_MD5HashA2AuthInt('sha', sha.sha)

    def test_MD5HashResponse(self, _algorithm='md5', _hash=md5.md5):
        """
        L{calcResponse} accepts the C{'md5'} algorithm and returns an MD5 hash
        of its parameters, excluding the nonce count, client nonce, and QoP
        value if the nonce count and client nonce are C{None}
        """
        hashA1 = 'abc123'
        hashA2 = '789xyz'
        nonce = 'lmnopq'

        response = '%s:%s:%s' % (hashA1, nonce, hashA2)
        expected = _hash(response).hexdigest()

        digest = calcResponse(hashA1, hashA2, _algorithm, nonce, None, None, None)
        self.assertEqual(expected, digest)

    def test_MD5SessionHashResponse(self):
        """
        L{calcResponse} accepts the C{'md5-sess'} algorithm and returns an MD5
        hash of its parameters, excluding the nonce count, client nonce, and
        QoP value if the nonce count and client nonce are C{None}
        """
        self.test_MD5HashResponse('md5-sess')

    def test_SHAHashResponse(self):
        """
        L{calcResponse} accepts the C{'sha'} algorithm and returns a SHA hash
        of its parameters, excluding the nonce count, client nonce, and QoP
        value if the nonce count and client nonce are C{None}
        """
        self.test_MD5HashResponse('sha', sha.sha)

    def test_MD5HashResponseExtra(self, _algorithm='md5', _hash=md5.md5):
        """
        L{calcResponse} accepts the C{'md5'} algorithm and returns an MD5 hash
        of its parameters, including the nonce count, client nonce, and QoP
        value if they are specified.
        """
        hashA1 = 'abc123'
        hashA2 = '789xyz'
        nonce = 'lmnopq'
        nonceCount = '00000004'
        clientNonce = 'abcxyz123'
        qop = 'auth'

        response = '%s:%s:%s:%s:%s:%s' % (
            hashA1, nonce, nonceCount, clientNonce, qop, hashA2)
        expected = _hash(response).hexdigest()

        digest = calcResponse(
            hashA1, hashA2, _algorithm, nonce, nonceCount, clientNonce, qop)
        self.assertEqual(expected, digest)

    def test_MD5SessionHashResponseExtra(self):
        """
        L{calcResponse} accepts the C{'md5-sess'} algorithm and returns an MD5
        hash of its parameters, including the nonce count, client nonce, and
        QoP value if they are specified.
        """
        self.test_MD5HashResponseExtra('md5-sess')

    def test_SHAHashResponseExtra(self):
        """
        L{calcResponse} accepts the C{'sha'} algorithm and returns a SHA hash
        of its parameters, including the nonce count, client nonce, and QoP
        value if they are specified.
        """
        self.test_MD5HashResponseExtra('sha', sha.sha)

    def makeRequest(self, method='GET', clientAddress=None):
        """
        Create a request object to be passed to
        L{basic.DigestCredentialFactory.decode} along with a response value.
        Override this in a subclass.
        """
        raise NotImplementedError("%r did not implement makeRequest" % (
                self.__class__,))

    def formatResponse(self, quotes=True, **kw):
        """
        Format all given keyword arguments and their values suitably for use as
        the value of an HTTP header.

        @types quotes: C{bool}
        @param quotes: A flag indicating whether to quote the values of each
            field in the response.

        @param **kw: Keywords and C{str} values which will be treated as field
            name/value pairs to include in the result.

        @rtype: C{str}
        @return: The given fields formatted for use as an HTTP header value.
        """
        if 'username' not in kw:
            kw['username'] = self.username
        if 'realm' not in kw:
            kw['realm'] = self.realm
        if 'algorithm' not in kw:
            kw['algorithm'] = self.algorithm
        if 'qop' not in kw:
            kw['qop'] = self.qop
        if 'cnonce' not in kw:
            kw['cnonce'] = self.cnonce
        if 'uri' not in kw:
            kw['uri'] = self.uri
        if quotes:
            quote = '"'
        else:
            quote = ''
        return ', '.join([
                '%s=%s%s%s' % (k, quote, v, quote)
                for (k, v)
                in kw.iteritems()
                if v is not None])

    def getDigestResponse(self, challenge, ncount):
        """
        Calculate the response for the given challenge
        """
        nonce = challenge.get('nonce')
        algo = challenge.get('algorithm').lower()
        qop = challenge.get('qop')

        ha1 = calcHA1(
            algo, self.username, self.realm, self.password, nonce, self.cnonce)
        ha2 = calcHA2(algo, "GET", self.uri, qop, None)
        expected = calcResponse(ha1, ha2, algo, nonce, ncount, self.cnonce, qop)
        return expected

    def test_interface(self):
        """
        L{DigestCredentialFactory} implements L{ICredentialFactory}.
        """
        self.assertTrue(
            verifyObject(ICredentialFactory, self.credentialFactory))

    def test_getChallenge(self):
        """
        The challenge issued by L{DigestCredentialFactory.getChallenge} must
        include C{'qop'}, C{'realm'}, C{'algorithm'}, C{'nonce'}, and
        C{'opaque'} keys.  The values for the C{'realm'} and C{'algorithm'}
        keys must match the values supplied to the factory's initializer.
        None of the values may have newlines in them.
        """
        challenge = self.credentialFactory.getChallenge(self.request)
        self.assertEquals(challenge['qop'], 'auth')
        self.assertEquals(challenge['realm'], 'test realm')
        self.assertEquals(challenge['algorithm'], 'md5')
        self.assertIn('nonce', challenge)
        self.assertIn('opaque', challenge)
        for v in challenge.values():
            self.assertNotIn('\n', v)

    def test_getChallengeWithoutClientIP(self):
        """
        L{DigestCredentialFactory.getChallenge} can issue a challenge even if
        the L{Request} it is passed returns C{None} from C{getClientIP}.
        """
        request = self.makeRequest('GET', None)
        challenge = self.credentialFactory.getChallenge(request)
        self.assertEqual(challenge['qop'], 'auth')
        self.assertEqual(challenge['realm'], 'test realm')
        self.assertEqual(challenge['algorithm'], 'md5')
        self.assertIn('nonce', challenge)
        self.assertIn('opaque', challenge)

    def test_response(self, quotes=True):
        """
        L{DigestCredentialFactory.decode} accepts a digest challenge response
        and parses it into an L{IUsernameHashedPassword} provider.
        """
        challenge = self.credentialFactory.getChallenge(self.request)

        nc = "00000001"
        clientResponse = self.formatResponse(
            quotes=quotes,
            nonce=challenge['nonce'],
            response=self.getDigestResponse(challenge, nc),
            nc=nc,
            opaque=challenge['opaque'])
        creds = self.credentialFactory.decode(
            clientResponse, self.request)
        self.assertTrue(creds.checkPassword(self.password))
        self.assertFalse(creds.checkPassword(self.password + 'wrong'))

    def test_responseWithoutQuotes(self):
        """
        L{DigestCredentialFactory.decode} accepts a digest challenge response
        which does not quote the values of its fields and parses it into an
        L{IUsernameHashedPassword} provider in the same way it would a
        response which included quoted field values.
        """
        self.test_response(False)

    def test_caseInsensitiveAlgorithm(self):
        """
        The case of the algorithm value in the response is ignored when
        checking the credentials.
        """
        self.algorithm = 'MD5'
        self.test_response()

    def test_md5DefaultAlgorithm(self):
        """
        The algorithm defaults to MD5 if it is not supplied in the response.
        """
        self.algorithm = None
        self.test_response()

    def test_responseWithoutClientIP(self):
        """
        L{DigestCredentialFactory.decode} accepts a digest challenge response
        even if the L{Request} it is passed returns C{None} from
        C{getClientIP}.
        """
        request = self.makeRequest('GET', None)
        challenge = self.credentialFactory.getChallenge(request)

        nc = "00000001"
        clientResponse = self.formatResponse(
            nonce=challenge['nonce'],
            response=self.getDigestResponse(challenge, nc),
            nc=nc,
            opaque=challenge['opaque'])
        creds = self.credentialFactory.decode(clientResponse, request)
        self.assertTrue(creds.checkPassword(self.password))
        self.assertFalse(creds.checkPassword(self.password + 'wrong'))

    def test_multiResponse(self):
        """
        L{DigestCredentialFactory.decode} handles multiple responses to a
        single challenge.
        """
        challenge = self.credentialFactory.getChallenge(self.request)

        nc = "00000001"
        clientResponse = self.formatResponse(
            nonce=challenge['nonce'],
            response=self.getDigestResponse(challenge, nc),
            nc=nc,
            opaque=challenge['opaque'])

        creds = self.credentialFactory.decode(clientResponse, self.request)
        self.assertTrue(creds.checkPassword(self.password))
        self.assertFalse(creds.checkPassword(self.password + 'wrong'))

        nc = "00000002"
        clientResponse = self.formatResponse(
            nonce=challenge['nonce'],
            response=self.getDigestResponse(challenge, nc),
            nc=nc,
            opaque=challenge['opaque'])

        creds = self.credentialFactory.decode(clientResponse, self.request)
        self.assertTrue(creds.checkPassword(self.password))
        self.assertFalse(creds.checkPassword(self.password + 'wrong'))

    def test_failsWithDifferentMethod(self):
        """
        L{DigestCredentialFactory.decode} returns an L{IUsernameHashedPassword}
        provider which rejects a correct password for the given user if the
        challenge response request is made using a different HTTP method than
        was used to request the initial challenge.
        """
        challenge = self.credentialFactory.getChallenge(self.request)

        nc = "00000001"
        clientResponse = self.formatResponse(
            nonce=challenge['nonce'],
            response=self.getDigestResponse(challenge, nc),
            nc=nc,
            opaque=challenge['opaque'])

        postRequest = self.makeRequest('POST', self.clientAddress)
        creds = self.credentialFactory.decode(clientResponse, postRequest)
        self.assertFalse(creds.checkPassword(self.password))
        self.assertFalse(creds.checkPassword(self.password + 'wrong'))

    def test_noUsername(self):
        """
        L{DigestCredentialFactory.decode} raises L{LoginFailed} if the response
        has no username field or if the username field is empty.
        """
        # Check for no username
        e = self.assertRaises(
            error.LoginFailed,
            self.credentialFactory.decode,
            self.formatResponse(username=None),
            self.request)
        self.assertEqual(str(e), "Invalid response, no username given.")

        # Check for an empty username
        e = self.assertRaises(
            error.LoginFailed,
            self.credentialFactory.decode,
            self.formatResponse(username=""),
            self.request)
        self.assertEqual(str(e), "Invalid response, no username given.")

    def test_noNonce(self):
        """
        L{DigestCredentialFactory.decode} raises L{LoginFailed} if the response
        has no nonce.
        """
        e = self.assertRaises(
            error.LoginFailed,
            self.credentialFactory.decode,
            self.formatResponse(opaque="abc123"),
            self.request)
        self.assertEqual(str(e), "Invalid response, no nonce given.")

    def test_noOpaque(self):
        """
        L{DigestCredentialFactory.decode} raises L{LoginFailed} if the response
        has no opaque.
        """
        e = self.assertRaises(
            error.LoginFailed,
            self.credentialFactory.decode,
            self.formatResponse(),
            self.request)
        self.assertEqual(str(e), "Invalid response, no opaque given.")

    def test_checkHash(self):
        """
        L{DigestCredentialFactory.decode} returns an L{IUsernameDigestHash}
        provider which can verify a hash of the form 'username:realm:password'.
        """
        challenge = self.credentialFactory.getChallenge(self.request)

        nc = "00000001"
        clientResponse = self.formatResponse(
            nonce=challenge['nonce'],
            response=self.getDigestResponse(challenge, nc),
            nc=nc,
            opaque=challenge['opaque'])

        creds = self.credentialFactory.decode(clientResponse, self.request)
        self.assertTrue(verifyObject(IUsernameDigestHash, creds))

        cleartext = '%s:%s:%s' % (self.username, self.realm, self.password)
        hash = md5.md5(cleartext)
        self.assertTrue(creds.checkHash(hash.hexdigest()))
        hash.update('wrong')
        self.assertFalse(creds.checkHash(hash.hexdigest()))

    def test_invalidOpaque(self):
        """
        L{DigestCredentialFactory.decode} raises L{LoginFailed} when the opaque
        value does not contain all the required parts.
        """
        credentialFactory = FakeDigestCredentialFactory(self.algorithm, self.realm)
        challenge = credentialFactory.getChallenge(self.request)

        exc = self.assertRaises(
            error.LoginFailed,
            credentialFactory.verifyOpaque,
            'badOpaque',
            challenge['nonce'],
            self.clientAddress.host)
        self.assertEqual(str(exc), 'Invalid response, invalid opaque value')

        badOpaque = 'foo-' + b64encode('nonce,clientip')

        exc = self.assertRaises(
            error.LoginFailed,
            credentialFactory.verifyOpaque,
            badOpaque,
            challenge['nonce'],
            self.clientAddress.host)
        self.assertEqual(str(exc), 'Invalid response, invalid opaque value')

        exc = self.assertRaises(
            error.LoginFailed,
            credentialFactory.verifyOpaque,
            '',
            challenge['nonce'],
            self.clientAddress.host)
        self.assertEqual(str(exc), 'Invalid response, invalid opaque value')

        badOpaque = (
            'foo-' + b64encode('%s,%s,foobar' % (
                    challenge['nonce'],
                    self.clientAddress.host)))
        exc = self.assertRaises(
            error.LoginFailed,
            credentialFactory.verifyOpaque,
            badOpaque,
            challenge['nonce'],
            self.clientAddress.host)
        self.assertEqual(
            str(exc), 'Invalid response, invalid opaque/time values')

    def test_incompatibleNonce(self):
        """
        L{DigestCredentialFactory.decode} raises L{LoginFailed} when the given
        nonce from the response does not match the nonce encoded in the opaque.
        """
        credentialFactory = FakeDigestCredentialFactory(self.algorithm, self.realm)
        challenge = credentialFactory.getChallenge(self.request)

        badNonceOpaque = credentialFactory.generateOpaque(
            '1234567890',
            self.clientAddress.host)

        exc = self.assertRaises(
            error.LoginFailed,
            credentialFactory.verifyOpaque,
            badNonceOpaque,
            challenge['nonce'],
            self.clientAddress.host)
        self.assertEqual(
            str(exc),
            'Invalid response, incompatible opaque/nonce values')

        exc = self.assertRaises(
            error.LoginFailed,
            credentialFactory.verifyOpaque,
            badNonceOpaque,
            '',
            self.clientAddress.host)
        self.assertEqual(
            str(exc),
            'Invalid response, incompatible opaque/nonce values')

    def test_incompatibleClientIP(self):
        """
        L{DigestCredentialFactory.decode} raises L{LoginFailed} when the
        request comes from a client IP other than what is encoded in the
        opaque.
        """
        credentialFactory = FakeDigestCredentialFactory(self.algorithm, self.realm)
        challenge = credentialFactory.getChallenge(self.request)

        badAddress = '10.0.0.1'
        # Sanity check
        self.assertNotEqual(self.clientAddress.host, badAddress)

        badNonceOpaque = credentialFactory.generateOpaque(
            challenge['nonce'], badAddress)

        self.assertRaises(
            error.LoginFailed,
            credentialFactory.verifyOpaque,
            badNonceOpaque,
            challenge['nonce'],
            self.clientAddress.host)

    def test_oldNonce(self):
        """
        L{DigestCredentialFactory.decode} raises L{LoginFailed} when the given
        opaque is older than C{DigestCredentialFactory.CHALLENGE_LIFETIME_SECS}
        """
        credentialFactory = FakeDigestCredentialFactory(self.algorithm, self.realm)
        challenge = credentialFactory.getChallenge(self.request)

        key = '%s,%s,%s' % (challenge['nonce'],
                            self.clientAddress.host,
                            '-137876876')
        digest = md5.md5(key + credentialFactory.privateKey).hexdigest()
        ekey = b64encode(key)

        oldNonceOpaque = '%s-%s' % (digest, ekey.strip('\n'))

        self.assertRaises(
            error.LoginFailed,
            credentialFactory.verifyOpaque,
            oldNonceOpaque,
            challenge['nonce'],
            self.clientAddress.host)

    def test_mismatchedOpaqueChecksum(self):
        """
        L{DigestCredentialFactory.decode} raises L{LoginFailed} when the opaque
        checksum fails verification.
        """
        credentialFactory = FakeDigestCredentialFactory(self.algorithm, self.realm)
        challenge = credentialFactory.getChallenge(self.request)

        key = '%s,%s,%s' % (challenge['nonce'],
                            self.clientAddress.host,
                            '0')

        digest = md5.md5(key + 'this is not the right pkey').hexdigest()
        badChecksum = '%s-%s' % (digest, b64encode(key))

        self.assertRaises(
            error.LoginFailed,
            credentialFactory.verifyOpaque,
            badChecksum,
            challenge['nonce'],
            self.clientAddress.host)

    def test_incompatibleCalcHA1Options(self):
        """
        L{calcHA1} raises L{TypeError} when any of the pszUsername, pszRealm,
        or pszPassword arguments are specified with the preHA1 keyword
        argument.
        """
        arguments = (
            ("user", "realm", "password", "preHA1"),
            (None, "realm", None, "preHA1"),
            (None, None, "password", "preHA1"),
            )

        for pszUsername, pszRealm, pszPassword, preHA1 in arguments:
            self.assertRaises(
                TypeError,
                calcHA1,
                "md5",
                pszUsername,
                pszRealm,
                pszPassword,
                "nonce",
                "cnonce",
                preHA1=preHA1)

class DigestAuthTestCase(RequestMixin, DigestAuthTestsMixin, unittest.TestCase):
    """
    Digest authentication tests which use L{twisted.web.http.Request}.
    """

def render(resource, request):
    result = resource.render(request)
    if result is NOT_DONE_YET:
        return
    request.write(result)
    request.finish()

class UnauthorizedResourceTests(unittest.TestCase):
    """
    Tests for L{UnauthorizedResource}.
    """
    def test_getChildWithDefault(self):
        """
        An L{UnauthorizedResource} is every child of itself.
        """
        resource = UnauthorizedResource([])
        self.assertIdentical(
            resource.getChildWithDefault("foo", None), resource)
        self.assertIdentical(
            resource.getChildWithDefault("bar", None), resource)

    def test_render(self):
        """
        L{UnauthorizedResource} renders with a 401 response code and a
        I{WWW-Authenticate} header and puts a simple unauthorized message
        into the response body.
        """
        resource = UnauthorizedResource([
                BasicCredentialFactory('example.com')])
        request = DummyRequest([''])
        render(resource, request)
        self.assertEqual(request.responseCode, 401)
        self.assertEqual(
            request.responseHeaders.getRawHeaders('www-authenticate'),
            ['basic realm="example.com"'])
        self.assertEqual(request.written, ['Unauthorized'])

    def test_renderQuotesRealm(self):
        """
        The realm value included in the I{WWW-Authenticate} header set in
        the response when L{UnauthorizedResounrce} is rendered has quotes
        and backslashes escaped.
        """
        resource = UnauthorizedResource([
                BasicCredentialFactory('example\\"foo')])
        request = DummyRequest([''])
        render(resource, request)
        self.assertEqual(
            request.responseHeaders.getRawHeaders('www-authenticate'),
            ['basic realm="example\\\\\\"foo"'])

class Realm(object):
    """
    A simple L{IRealm} implementation which gives out L{WebAvatar} for any
    avatarId.

    @type loggedIn: C{int}
    @ivar loggedIn: The number of times C{requestAvatar} has been invoked for
        L{IResource}.

    @type loggedOut: C{int}
    @ivar loggedOut: The number of times the logout callback has been invoked.
    """
    implements(portal.IRealm)

    def __init__(self, avatarFactory):
        self.loggedOut = 0
        self.loggedIn = 0
        self.avatarFactory = avatarFactory

    def requestAvatar(self, avatarId, mind, *interfaces):
        if IResource in interfaces:
            self.loggedIn += 1
            return IResource, self.avatarFactory(avatarId), self.logout
        raise NotImplementedError()

    def logout(self):
        self.loggedOut += 1

class HTTPAuthHeaderTests(unittest.TestCase):
    """
    Tests for L{HTTPAuthSessionWrapper}.
    """
    makeRequest = DummyRequest

    def setUp(self):
        """
        Create a realm, portal, and L{HTTPAuthSessionWrapper} to use in the tests.
        """
        self.username = 'foo bar'
        self.password = 'bar baz'
        self.childName = "foo-child"
        self.childContent = "contents of the foo child of the avatar"
        self.checker = InMemoryUsernamePasswordDatabaseDontUse()
        self.checker.addUser(self.username, self.password)
        self.avatar = Resource()
        self.avatar.putChild(
            self.childName, Data(self.childContent, 'text/plain'))
        self.avatars = {self.username: self.avatar}
        self.realm = Realm(self.avatars.get)
        self.portal = portal.Portal(self.realm, [self.checker])
        self.credentialFactories = []
        self.wrapper = HTTPAuthSessionWrapper(
            self.portal, self.credentialFactories)

    def _authorizedBasicLogin(self, request, creds=None):
        if creds is None:
            username, password = self.username, self.password
        else:
            username, password = creds
        authorization = b64encode(username + ':' + password)
        request.headers['authorization'] = 'Basic ' + authorization
        child = self.wrapper.getChildWithDefault(request.postpath[0], request)
        return child

    def test_getChildWithDefault(self):
        """
        L{HTTPAuthSessionWrapper.getChildWithDefault} returns an
        L{UnauthorizedResource} instance when called with a request which does
        not have the required I{Authorization} headers.
        """
        request = self.makeRequest([self.childName])
        child = self.wrapper.getChildWithDefault(self.childName, request)
        self.assertIsInstance(child, UnauthorizedResource)

    def _invalidAuthorizationTest(self, response):
        self.credentialFactories.append(BasicCredentialFactory('example.com'))
        request = self.makeRequest([self.childName])
        request.headers['authorization'] = response
        child = self.wrapper.getChildWithDefault(self.childName, request)
        d = request.notifyFinish()
        def cbFinished(result):
            self.assertEqual(request.responseCode, 401)
        d.addCallback(cbFinished)
        render(child, request)
        return d

    def test_getChildWithDefaultUnauthorizedUser(self):
        """
        If L{HTTPAuthSessionWrapper.getChildWithDefault} is called with a
        request with an I{Authorization} header with a user which does not
        exist, an L{IResource} which renders a 401 response code is returned.
        """
        return self._invalidAuthorizationTest('Basic ' + b64encode('foo:bar'))

    def test_getChildWithDefaultUnauthorizedPassword(self):
        """
        If L{HTTPAuthSessionWrapper.getChildWithDefault} is called with a
        request with an I{Authorization} header with a user which exists and
        the wrong password, an L{IResource} which renders a 401 response code
        is returned.
        """
        return self._invalidAuthorizationTest(
            'Basic ' + b64encode(self.username + ':bar'))

    def test_getChildWithDefaultUnrecognizedScheme(self):
        """
        If L{HTTPAuthSessionWrapper.getChildWithDefault} is called with a
        request with an I{Authorization} header with an unrecognized scheme, an
        L{IResource} which renders a 401 response code is returned.
        """
        return self._invalidAuthorizationTest('Quux foo bar baz')

    def test_getChildWithDefaultAuthorized(self):
        """
        When called with a request with a valid I{Authorization} header,
        L{HTTPAuthSessionWrapper.getChildWithDefault} returns an L{IResource}
        which renders the L{IResource} avatar retrieved from the portal.
        """
        self.credentialFactories.append(BasicCredentialFactory('example.com'))
        request = self.makeRequest([self.childName])
        child = self._authorizedBasicLogin(request)
        d = request.notifyFinish()
        def cbFinished(ignored):
            self.assertEquals(request.written, [self.childContent])
        d.addCallback(cbFinished)
        render(child, request)
        return d

    def test_getChallengeCalledWithRequest(self):
        """
        When L{HTTPAuthSessionWrapper} finds an L{ICredentialFactory} to issue
        a challenge, it calls the C{getChallenge} method with the request as an
        argument.
        """
        class DumbCredentialFactory(object):
            implements(ICredentialFactory)
            scheme = 'dumb'

            def __init__(self):
                self.requests = []

            def getChallenge(self, request):
                self.requests.append(request)
                return {}

        factory = DumbCredentialFactory()
        self.credentialFactories.append(factory)
        request = self.makeRequest([self.childName])
        child = self.wrapper.getChildWithDefault(request.postpath[0], request)
        d = request.notifyFinish()
        def cbFinished(ignored):
            self.assertEqual(factory.requests, [request])
        d.addCallback(cbFinished)
        render(child, request)
        return d

    def test_logout(self):
        """
        The realm's logout callback is invoked after the resource is rendered.
        """
        self.credentialFactories.append(BasicCredentialFactory('example.com'))

        class SlowerResource(Resource):
            def render(self, request):
                return NOT_DONE_YET

        self.avatar.putChild(self.childName, SlowerResource())
        request = self.makeRequest([self.childName])
        child = self._authorizedBasicLogin(request)
        render(child, request)
        self.assertEqual(self.realm.loggedOut, 0)
        request.finish()
        self.assertEqual(self.realm.loggedOut, 1)

    def test_decodeRaises(self):
        """
        L{HTTPAuthSessionWrapper.getChildWithDefault} returns an
        L{UnauthorizedResource} instance when called with a request which has a
        I{Basic Authorization} header which cannot be decoded using base64.
        """
        self.credentialFactories.append(BasicCredentialFactory('example.com'))
        request = self.makeRequest([self.childName])
        request.headers['authorization'] = 'Basic decode should fail'
        child = self.wrapper.getChildWithDefault(self.childName, request)
        self.assertIsInstance(child, UnauthorizedResource)

    def test_selectParseResponse(self):
        """
        L{HTTPAuthSessionWrapper._selectParseHeader} returns a two-tuple giving
        the L{ICredentialFactory} to use to parse the header and a string
        containing the portion of the header which remains to be parsed.
        """
        basicAuthorization = 'Basic abcdef123456'
        self.assertEqual(
            self.wrapper._selectParseHeader(basicAuthorization),
            (None, None))
        factory = BasicCredentialFactory('example.com')
        self.credentialFactories.append(factory)
        self.assertEqual(
            self.wrapper._selectParseHeader(basicAuthorization),
            (factory, 'abcdef123456'))

    def test_unexpectedDecodeError(self):
        """
        Any unexpected exception raised by the credential factory's C{decode}
        method results in a 500 response code and causes the exception to be
        logged.
        """
        class UnexpectedException(Exception):
            pass

        class BadFactory(object):
            scheme = 'bad'

            def getChallenge(self, client):
                return {}

            def decode(self, response, request):
                raise UnexpectedException()

        self.credentialFactories.append(BadFactory())
        request = self.makeRequest([self.childName])
        request.headers['authorization'] = 'Bad abc'
        child = self.wrapper.getChildWithDefault(self.childName, request)
        render(child, request)
        self.assertEqual(request.responseCode, 500)
        self.assertEqual(len(self.flushLoggedErrors(UnexpectedException)), 1)

    def test_unexpectedLoginError(self):
        """
        Any unexpected failure from L{Portal.login} results in a 500 response
        code and causes the failure to be logged.
        """
        class UnexpectedException(Exception):
            pass

        class BrokenChecker(object):
            credentialInterfaces = (IUsernamePassword,)

            def requestAvatarId(self, credentials):
                raise UnexpectedException()

        self.portal.registerChecker(BrokenChecker())
        self.credentialFactories.append(BasicCredentialFactory('example.com'))
        request = self.makeRequest([self.childName])
        child = self._authorizedBasicLogin(request)
        render(child, request)
        self.assertEqual(request.responseCode, 500)
        self.assertEqual(len(self.flushLoggedErrors(UnexpectedException)), 1)