text:
package org.xlcloud.openstack.client;

import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
import java.util.List;

import javax.ws.rs.core.MediaType;

import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.xlcloud.logging.LoggingUtils;
import org.xlcloud.openstack.api.IdentityManagementClient;
import org.xlcloud.openstack.api.identity.IdentityAdministrationEndpoint;
import org.xlcloud.openstack.api.identity.UserResource;
import org.xlcloud.openstack.api.identity.UsersResource;
import org.xlcloud.openstack.model.exceptions.OpenStackAuthenticationException;
import org.xlcloud.openstack.model.exceptions.OpenStackNotFoundException;
import org.xlcloud.openstack.model.identity.Access;
import org.xlcloud.openstack.model.identity.Role;
import org.xlcloud.openstack.model.identity.RoleList;
import org.xlcloud.openstack.model.identity.Tenant;
import org.xlcloud.openstack.model.identity.TenantList;
import org.xlcloud.openstack.model.identity.Token;
import org.xlcloud.openstack.model.identity.User;
import org.xlcloud.openstack.model.identity.UserForCreate;
import org.xlcloud.openstack.model.identity.UserList;
import org.xlcloud.openstack.model.identity.keystone.KeystoneAccess;
import org.xlcloud.openstack.model.identity.keystone.KeystoneAuthentication;
import org.xlcloud.openstack.rest.PerformanceMonitorFilter;
import org.xlcloud.rest.client.config.ClientLoggingFilter;

import com.sun.jersey.api.client.Client;

/**
 * This class is the main implementation of the {@link IdentityManagementClient}
 * . As there is no public constructor of this class, the only way to retrieve
 * the instance of this class is by invoking one of static methods getInstance.
 * To see the difference between them, compare
 * {@link OpenStackKeystoneClient#getInstance(String,String,String,String)} and
 * {@link OpenStackKeystoneClient#getInstance(Token, String)}.
 *
 * @author Andrzej Stasiak, AMG.net
 */
public class OpenStackKeystoneClient implements IdentityManagementClient {

    private static final Logger LOG = Logger.getLogger(OpenStackKeystoneClient.class);

    private final String resourceUri;

    private final Client client;

    private String tokenId;

    private OpenStackKeystoneClient( String endpoint ) {
        this.client = KeystoneRestClientProducer.INSTANCE.produceClient();
        this.resourceUri = endpoint;

        LOG.debug("Using Jersey client " + client.hashCode());
    }

    /**
     * This method creates and returns new object of OpenStackClient class. It
     * takes as parameters following items: user credentials (username and
     * password), name of the tenant the user should be logged on (one user may
     * belong to many tenants, and their privileges may differ between tenants)
     * and endpoint URI indicating the location of Keystone server (including
     * port number).
     *

     * The return value is not instance of OpenStackClient, but instead
     * {@link java.lang.reflect.Proxy} with {@link RefreshAuthenticationHandler}
     * . Note that, therefore, authentication on openstack is not performed at
     * the time of creating the client, but during the first request. This
     * behaviour is different than behavior of
     * {@link OpenStackKeystoneClient#getInstance(String, String)}.
     *

     *
     * @param username
     *            name of user to be logged as
     * @param password
     *            password of user to be logged as
     * @param tenantName
     *            name of tenant to be logged in
     * @param keystoneEndpointUri
     *            an absolute URI to Keystone server
     * @return instance of OpenStackClient
     */
    public static IdentityManagementClient getKeystoneClientInstance(String username, String password, String tenantName, String endpointUri) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Creating new OpenStackClient for user " + username + " on tenant " + tenantName + ". Endpoint URI: " + endpointUri);
        }

        OpenStackKeystoneClient osClient = new OpenStackKeystoneClient(endpointUri);
        osClient.setFilters();
        InvocationHandler handler = new RefreshAuthenticationHandler(osClient, username, password, tenantName);
        IdentityManagementClient imClient = (IdentityManagementClient) Proxy.newProxyInstance(
                OpenStackKeystoneClient.class.getClassLoader(), new Class[] { IdentityManagementClient.class }, handler);
        return imClient;
    }

    /**
     * This method creates and returns new object of OpenStackClient class. It
     * takes as parameters keystone token and endpoint URI indicating the
     * location of Keystone server (including port number). In contrary to
     * {@link OpenStackKeystoneClient#getInstance(String,String,String,String)}
     * this method returns real OpenStackClient object, without proxy.
     *

     * This is not default way of obtaining OpenStackClient. You should use it
     * only when you want to connect with existing Keystone session (e.g. super
     * as Keystone superuser, who has fixed token and no user credentials).
     *

     * When given endpointUri does not indicate Keystone server,
     * {@link OpenStackNotFoundException} is thrown. If given token id is not
     * valid, {@link OpenStackAuthenticationException} is thrown.
     *

     * As an implementation of {@link IdentityManagementClient} it uses set of
     * implementations of identity interfaces (package
     * org.xlcloud.openstack.model.identity) from package
     * org.xlcloud.openstack.model.identity.keystone.
     *
     * @param token
     *            Keystone token id
     * @param endpointUri
     *            an absolute URI to Keystone server
     * @return
     */
    public static IdentityManagementClient getKeystoneClientInstance(String token, String endpointUri) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Creating new OpenStackClient with token: " + LoggingUtils.maskPartially(token) + ". Endpoint URI: " + endpointUri);
        }

        OpenStackKeystoneClient osClient = new OpenStackKeystoneClient(endpointUri);
        osClient.setToken(token);

        return osClient;
    }

    /** {@inheritDoc} */
    public void reauthenticate(String username, String password, String tenantName) {
        if (isTokenValid(tokenId)) {
            // throw new OpenStackAuthenticationException("Current token is valid");
            LOG.warn("Current token is valid, but we'll reauthenticate the user anyway.");
        }
        KeystoneAuthentication authentication = KeystoneAuthentication.withPasswordCredentials(username, password);
        if (StringUtils.isNotBlank(tenantName)) {
            authentication.setTenantName(tenantName);
        }
        Access access = client.resource(resourceUri + "/tokens").type(MediaType.APPLICATION_JSON).accept(MediaType.APPLICATION_JSON)
                .post(KeystoneAccess.class, authentication);
        tokenId = access.getToken().getId();
        setFilters();
    }

    /**
     * This method should not be invoked directly, it is invoked by
     * RefreshAuthenticationHandler.
     *
     * @param id
     *            token id to be used
     */
    void setToken(String id) {
        this.tokenId = id;
        setFilters();
    }

    /**
     * This method is made for testing purposes. It should not be invoked when
     * using OpenStackClient.
     *
     * @return token of current Keystone session.
     */
    @Override
    public String getToken() {
        return this.tokenId;
    }

    private void setFilters() {
        client.removeAllFilters();
        if (tokenId != null) {
            client.addFilter(new AuthFilter(tokenId));
        }
        client.addFilter(new ClientLoggingFilter());
        client.addFilter(new ExceptionHandlingFilter());
        client.addFilter(new PerformanceMonitorFilter());
    }

    private IdentityAdministrationEndpoint getIdentityAdministationEndpoint() {
        return new IdentityAdministrationEndpoint(client, resourceUri);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public boolean isTokenValid(String token) {
        if (StringUtils.isBlank(token)) {
            return false;
        }

        try {
            String digest = DigestUtils.md5Hex(token);
            LOG.debug("Transformed token [" + LoggingUtils.maskPartially(token) + "] to digest [" + digest + "]");
            getIdentityAdministationEndpoint().tokens().token(digest).get();
        } catch (OpenStackAuthenticationException | OpenStackNotFoundException ex) {
            return false;
        }
        return true;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public UserList getUserResources() {
        return getIdentityAdministationEndpoint().users().get();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public User getUserById(String id) {
        return getIdentityAdministationEndpoint().users().user(id).get();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public User getUserByName(String name) throws OpenStackNotFoundException {
        return getIdentityAdministationEndpoint().users().getUserByName(name);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void deleteUserById(String id) {
        getIdentityAdministationEndpoint().users().user(id).delete();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public User createUser(UserForCreate user) {
        IdentityAdministrationEndpoint identityAdministationEndpoint = getIdentityAdministationEndpoint();
        UsersResource users = identityAdministationEndpoint.users();
        return users.post(user);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public User updateUser(UserForCreate user) {
        IdentityAdministrationEndpoint identityAdministationEndpoint = getIdentityAdministationEndpoint();
        UserResource userRespource = identityAdministationEndpoint.users().user(user.getId());
        return userRespource.put(user);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public Tenant createTentant(Tenant tenant) {
        return getIdentityAdministationEndpoint().tenants().post(tenant);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public Role createRole(Role role) {
        return getIdentityAdministationEndpoint().roles().post(role);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public TenantList getTenantResources() {
        return getIdentityAdministationEndpoint().tenants().get();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public Role assignRole(String userId, String roleId, String tenantId) {
        return getIdentityAdministationEndpoint().tenants().tenant(tenantId).users().user(userId).roles().role(roleId).put();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void removeRole(String userId, String roleId, String tenantId) {
        getIdentityAdministationEndpoint().tenants().tenant(tenantId).users().user(userId).roles().role(roleId).delete();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public RoleList getRoleResources() {
        return getIdentityAdministationEndpoint().roles().get();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public Tenant getTenantByName(String name) throws OpenStackNotFoundException {
        return getIdentityAdministationEndpoint().tenants().getTenantByName(name);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public Role getRoleByName(String name) throws OpenStackNotFoundException {
        /*
         * According to the documentation
         * (http://docs.openstack.org/api/openstack
         * -identity-service/2.0/content/
         * Admin_API_Service_Developer_Operations-d1e1357.html), it should be
         * possible to get role by name in the same manner as in getUserByName()
         * and getTentantByName(). However for some reason the "name" parameter
         * is ignored and the whole list is returned.
         */
        RoleList roleList = getRoleResources();
        if (roleList != null) {
            List<Role> roles = roleList.getList();
            for (Role role : roles) {
                if (name.equals(role.getName())) {
                    return role;
                }
            }
        }
        throw new OpenStackNotFoundException("Could not find role: " + name);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public Tenant getTenantById(String id) {
        return getIdentityAdministationEndpoint().tenants().tenant(id).get();
    }

    @Override
    public void removeTenant(String id) {
        getIdentityAdministationEndpoint().tenants().tenant(id).delete();
    }

    /** {@inheritDoc} */
    @Override
    public RoleList getUserRolesInTenant(String userId, String tenantId) {
        return getIdentityAdministationEndpoint().tenants().tenant(tenantId).users().user(userId).roles().get();
    }

    /** {@inheritDoc} */
    @Override
    public void removeAllUserRolesInTenant(String userId, String tenantId) {
        RoleList roleList = getUserRolesInTenant(userId, tenantId);
        if (roleList == null) {
            return;
        }
        for (Role role : roleList.getList()) {
            removeRole(userId, role.getId(), tenantId);
        }
    }
}


text:
package com.dianping.cat.report.page.cache;

import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Set;

import org.unidal.web.mvc.view.annotation.EntityMeta;
import org.unidal.web.mvc.view.annotation.ModelMeta;

import com.dianping.cat.helper.SortHelper;
import com.dianping.cat.mvc.AbstractReportModel;
import com.dianping.cat.report.ReportPage;

@ModelMeta("cache")
public class Model extends AbstractReportModel<Action, ReportPage, Context> {
	private String m_queryName;

	@EntityMeta
	private CacheReport m_report;

	private String m_pieChart;

	public Model(Context ctx) {
		super(ctx);
	}

	@Override
	public Action getDefaultAction() {
		return Action.HISTORY_REPORT;
	}

	@Override
	public String getDomain() {
		return m_report.getDomain();
	}

	@Override
	public Collection<String> getDomains() {
		if (m_report == null) {
			ArrayList<String> arrayList = new ArrayList<String>();
			arrayList.add(getDomain());

			return arrayList;
		} else {
			Set<String> domainNames = m_report.getDomainNames();

			return SortHelper.sortDomain(domainNames);
		}
	}

	public List<String> getIps() {
		if (m_report == null) {
			return new ArrayList<String>();
		} else {
			return SortHelper.sortDomain(m_report.getIps());
		}
	}

	public String getPieChart() {
		return m_pieChart;
	}

	public String getQueryName() {
		return m_queryName;
	}

	public CacheReport getReport() {
		return m_report;
	}

	public void setPieChart(String pieChart) {
		m_pieChart = pieChart;
	}

	public void setQueryName(String queryName) {
		m_queryName = queryName;
	}

	public void setReport(CacheReport report) {
		m_report = report;
	}

}

redpajama_set_name: RedPajamaGithub


text:
package xds

import (
	"fmt"
	"reflect"
	"sort"
	"strconv"
	"sync"
	"testing"
	"time"

	"istio.io/istio/pilot/pkg/model"
	"istio.io/istio/pkg/config/schema/kind"
	"istio.io/istio/pkg/util/sets"
	"istio.io/istio/tests/util/leak"
)

// Helper function to remove an item or timeout and return nil if there are no pending pushes
func getWithTimeout(p *PushQueue) *Connection {
	done := make(chan *Connection, 1)
	go func() {
		con, _, _ := p.Dequeue()
		done <- con
	}()
	select {
	case ret := <-done:
		return ret
	case <-time.After(time.Millisecond * 500):
		return nil
	}
}

func ExpectTimeout(t *testing.T, p *PushQueue) {
	t.Helper()
	done := make(chan struct{}, 1)
	go func() {
		p.Dequeue()
		done <- struct{}{}
	}()
	select {
	case <-done:
		t.Fatalf("Expected timeout")
	case <-time.After(time.Millisecond * 500):
	}
}

func ExpectDequeue(t *testing.T, p *PushQueue, expected *Connection) {
	t.Helper()
	result := make(chan *Connection, 1)
	go func() {
		con, _, _ := p.Dequeue()
		result <- con
	}()
	select {
	case got := <-result:
		if got != expected {
			t.Fatalf("Expected proxy %v, got %v", expected, got)
		}
	case <-time.After(time.Millisecond * 500):
		t.Fatalf("Timed out")
	}
}

func TestProxyQueue(t *testing.T) {
	proxies := make([]*Connection, 0, 100)
	for p := 0; p < 100; p++ {
		proxies = append(proxies, &Connection{conID: fmt.Sprintf("proxy-%d", p)})
	}

	t.Run("simple add and remove", func(t *testing.T) {
		t.Parallel()
		p := NewPushQueue()
		defer p.ShutDown()
		p.Enqueue(proxies[0], &model.PushRequest{})
		p.Enqueue(proxies[1], &model.PushRequest{})

		ExpectDequeue(t, p, proxies[0])
		ExpectDequeue(t, p, proxies[1])
	})

	t.Run("remove too many", func(t *testing.T) {
		t.Parallel()
		p := NewPushQueue()
		defer p.ShutDown()

		p.Enqueue(proxies[0], &model.PushRequest{})

		ExpectDequeue(t, p, proxies[0])
		ExpectTimeout(t, p)
	})

	t.Run("add multiple times", func(t *testing.T) {
		t.Parallel()
		p := NewPushQueue()
		defer p.ShutDown()

		p.Enqueue(proxies[0], &model.PushRequest{})
		p.Enqueue(proxies[1], &model.PushRequest{})
		p.Enqueue(proxies[0], &model.PushRequest{})

		ExpectDequeue(t, p, proxies[0])
		ExpectDequeue(t, p, proxies[1])
		ExpectTimeout(t, p)
	})

	t.Run("add and remove and markdone", func(t *testing.T) {
		t.Parallel()
		p := NewPushQueue()
		defer p.ShutDown()

		p.Enqueue(proxies[0], &model.PushRequest{})
		ExpectDequeue(t, p, proxies[0])
		p.MarkDone(proxies[0])
		p.Enqueue(proxies[0], &model.PushRequest{})
		ExpectDequeue(t, p, proxies[0])
		ExpectTimeout(t, p)
	})

	t.Run("add and remove and add and markdone", func(t *testing.T) {
		t.Parallel()
		p := NewPushQueue()
		defer p.ShutDown()

		p.Enqueue(proxies[0], &model.PushRequest{})
		ExpectDequeue(t, p, proxies[0])
		p.Enqueue(proxies[0], &model.PushRequest{})
		p.Enqueue(proxies[0], &model.PushRequest{})
		p.MarkDone(proxies[0])

		ExpectDequeue(t, p, proxies[0])
		ExpectTimeout(t, p)
	})

	t.Run("remove should block", func(t *testing.T) {
		t.Parallel()
		p := NewPushQueue()
		defer p.ShutDown()

		wg := &sync.WaitGroup{}
		wg.Add(1)
		go func() {
			ExpectDequeue(t, p, proxies[0])
			wg.Done()
		}()
		time.Sleep(time.Millisecond * 50)
		p.Enqueue(proxies[0], &model.PushRequest{})
		wg.Wait()
	})

	t.Run("should merge model.PushRequest", func(t *testing.T) {
		t.Parallel()
		p := NewPushQueue()
		defer p.ShutDown()

		firstTime := time.Now()
		p.Enqueue(proxies[0], &model.PushRequest{
			Full: false,
			ConfigsUpdated: sets.New(model.ConfigKey{
				Kind: kind.ServiceEntry,
				Name: "foo",
			}),
			Start: firstTime,
		})

		p.Enqueue(proxies[0], &model.PushRequest{
			Full:           false,
			ConfigsUpdated: sets.New(model.ConfigKey{Kind: kind.ServiceEntry, Name: "bar", Namespace: "ns1"}),

			Start: firstTime.Add(time.Second),
		})
		_, info, _ := p.Dequeue()

		if info.Start != firstTime {
			t.Errorf("Expected start time to be %v, got %v", firstTime, info.Start)
		}
		expectedEds := sets.New(
			model.ConfigKey{
				Kind:      kind.ServiceEntry,
				Name:      "foo",
				Namespace: "",
			},
			model.ConfigKey{
				Kind:      kind.ServiceEntry,
				Name:      "bar",
				Namespace: "ns1",
			},
		)
		if !reflect.DeepEqual(model.ConfigsOfKind(info.ConfigsUpdated, kind.ServiceEntry), expectedEds) {
			t.Errorf("Expected EdsUpdates to be %v, got %v", expectedEds, model.ConfigsOfKind(info.ConfigsUpdated, kind.ServiceEntry))
		}
		if info.Full {
			t.Errorf("Expected full to be false, got true")
		}
	})

	t.Run("two removes, one should block one should return", func(t *testing.T) {
		t.Parallel()
		p := NewPushQueue()
		defer p.ShutDown()

		wg := &sync.WaitGroup{}
		wg.Add(2)
		respChannel := make(chan *Connection, 2)
		go func() {
			respChannel <- getWithTimeout(p)
			wg.Done()
		}()
		time.Sleep(time.Millisecond * 50)
		p.Enqueue(proxies[0], &model.PushRequest{})
		go func() {
			respChannel <- getWithTimeout(p)
			wg.Done()
		}()

		wg.Wait()
		timeouts := 0
		close(respChannel)
		for resp := range respChannel {
			if resp == nil {
				timeouts++
			}
		}
		if timeouts != 1 {
			t.Fatalf("Expected 1 timeout, got %v", timeouts)
		}
	})

	t.Run("concurrent", func(t *testing.T) {
		t.Parallel()
		p := NewPushQueue()
		defer p.ShutDown()

		key := func(p *Connection, eds string) string { return fmt.Sprintf("%s~%s", p.conID, eds) }

		// We will trigger many pushes for eds services to each proxy. In the end we will expect
		// all of these to be dequeue, but order is not deterministic.
		expected := map[string]struct{}{}
		for eds := 0; eds < 100; eds++ {
			for _, pr := range proxies {
				expected[key(pr, fmt.Sprintf("%d", eds))] = struct{}{}
			}
		}
		go func() {
			for eds := 0; eds < 100; eds++ {
				for _, pr := range proxies {
					p.Enqueue(pr, &model.PushRequest{
						ConfigsUpdated: sets.New(model.ConfigKey{
							Kind: kind.ServiceEntry,
							Name: fmt.Sprintf("%d", eds),
						}),
					})
				}
			}
		}()

		done := make(chan struct{})
		mu := sync.RWMutex{}
		go func() {
			for {
				con, info, shuttingdown := p.Dequeue()
				if shuttingdown {
					return
				}
				for eds := range model.ConfigNamesOfKind(info.ConfigsUpdated, kind.ServiceEntry) {
					mu.Lock()
					delete(expected, key(con, eds))
					mu.Unlock()
				}
				p.MarkDone(con)
				if len(expected) == 0 {
					done <- struct{}{}
				}
			}
		}()

		select {
		case <-done:
		case <-time.After(time.Second * 10):
			mu.RLock()
			defer mu.RUnlock()
			t.Fatalf("failed to get all updates, still pending: %v", len(expected))
		}
	})

	t.Run("concurrent with deterministic order", func(t *testing.T) {
		t.Parallel()
		p := NewPushQueue()
		defer p.ShutDown()
		con := &Connection{conID: "proxy-test"}

		// We will trigger many pushes for eds services to the proxy. In the end we will expect
		// all of these to be dequeue, but order is deterministic.
		expected := make([]string, 100)
		for eds := 0; eds < 100; eds++ {
			expected[eds] = fmt.Sprintf("%d", eds)
		}
		go func() {
			// send to pushQueue
			for eds := 0; eds < 100; eds++ {
				p.Enqueue(con, &model.PushRequest{
					ConfigsUpdated: sets.New(model.ConfigKey{
						Kind: kind.Kind(eds),
						Name: fmt.Sprintf("%d", eds),
					}),
				})
			}
		}()

		processed := make([]string, 0, 100)
		done := make(chan struct{})
		pushChannel := make(chan *model.PushRequest)
		go func() {
			// dequeue pushQueue and send to pushChannel
			for {
				_, request, shuttingdown := p.Dequeue()
				if shuttingdown {
					close(pushChannel)
					return
				}
				pushChannel <- request
			}
		}()

		go func() {
			// recv from pushChannel and simulate push
			for {
				request := <-pushChannel
				if request == nil {
					return
				}
				updated := make([]string, 0, len(request.ConfigsUpdated))
				for configkey := range request.ConfigsUpdated {
					updated = append(updated, fmt.Sprintf("%d", configkey.Kind))
				}
				sort.Slice(updated, func(i, j int) bool {
					l, _ := strconv.Atoi(updated[i])
					r, _ := strconv.Atoi(updated[j])
					return l < r
				})
				processed = append(processed, updated...)
				if len(processed) == 100 {
					done <- struct{}{}
				}
				p.MarkDone(con)
			}
		}()

		select {
		case <-done:
		case <-time.After(time.Second * 10):
			t.Fatalf("failed to get all updates, still pending:  got %v", len(processed))
		}

		if !reflect.DeepEqual(expected, processed) {
			t.Fatalf("expected order %v, but got %v", expected, processed)
		}
	})
}

// TestPushQueueLeak is a regression test for https://github.com/grpc/grpc-go/issues/4758
func TestPushQueueLeak(t *testing.T) {
	ds := NewFakeDiscoveryServer(t, FakeOptions{})
	p := ds.ConnectADS()
	p.RequestResponseAck(t, nil)
	for _, c := range ds.Discovery.AllClients() {
		leak.MustGarbageCollect(t, c)
	}
	ds.Discovery.startPush(&model.PushRequest{})
	p.Cleanup()
}


text:
import argparse
from PIL import Image
import sys

def parse_args():
    parser = argparse.ArgumentParser(description='Resizes an image')
    parser.add_argument('--source', type=argparse.FileType('rb'), required=True)
    parser.add_argument('--width', type=str, required=True)
    parser.add_argument('--height', type=str, required=True)
    parser.add_argument('--x-offset', type=str, required=True)
    parser.add_argument('--y-offset', type=str, required=True)
    parser.add_argument('--output', type=argparse.FileType('wb+'), required=True)
    return parser.parse_args()

def parse_number(number):
    if number.endswith('%'):
        return float(float(number[:-1]) / 100)
    else:
        return int(number)

def check_bounds(number):
    if number < 0:
        print('Argument may not be negative: %d' % number)
        sys.exit(1)

    num_type = type(number)
    if num_type == float:
        if number > 1:
            print('Percent may not be greater than 100%')
            sys.exit(1)
    elif num_type == int:
        pass
    else:
        print('Invalid number type: %s' % num_type)
        sys.exit(1)

def is_pct(number):
    return type(number) == float

def convert_dimension(source_dimen, dest_dimen):
    if is_pct(dest_dimen):
        return int(source_dimen * dest_dimen)
    else:
        return dest_dimen

def main():
    args = parse_args()

    width = parse_number(args.width)
    height = parse_number(args.height)
    x_offset = parse_number(args.x_offset)
    y_offset = parse_number(args.y_offset)

    for value in [ width, height, x_offset, y_offset ]:
        check_bounds(value)

    with args.source as input_file:
        image = Image.open(input_file)

        image_width, image_height = image.size

        width = convert_dimension(image_width, width)
        height = convert_dimension(image_height, height)
        x_offset = convert_dimension(image_width, x_offset)
        y_offset = convert_dimension(image_height, y_offset)

        if x_offset + width > image_width:
            print('Cropped image outside bounds of source width')
            sys.exit(2)
        elif y_offset + height > image_height:
            print('Cropped image outside bounds of source height')
            sys.exit(2)

        resized_image = image.crop((x_offset, y_offset, x_offset + width, y_offset + height))
        with args.output as output_file:
            resized_image.save(output_file, format=image.format, mode=image.mode)

if __name__ == '__main__':
    main()


text:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">

<TITLE>Macromedia Flash Player Not Installed</TITLE>

  <META http-equiv="Content-Script-Type" content="text/javascript">

You don't have Macromedia Flash Player!

This site makes use of Macromedia®
          FlashTM software. To experience
          it the way we intended, you need Macromedia Flash Player, a plug-in for your Web
          Browser that can play the content we've created.

Why not download and install
          it now? It's easy, and you'll be joining more than 222 million people
          who can experience musical tracks, sound effects and animations integrated
          with innovative interfaces.

        <FORM NAME="preferences">

If you'd prefer not to
            install right now, you can still visit our alternate site, which does not require Macromedia Flash Player.

            <INPUT TYPE=checkbox NAME="demur">
            Check here if you want to bypass this notice and go
            directly to our alternate site on future visits.
        </FORM>

Macromedia and Flash
    are trademarks of Macromedia, Inc.


text:
from django.db import models
from django.contrib.auth.models import User

class Member(models.Model):
    user = models.OneToOneField(User)

    uuid = models.CharField('UID', max_length=100)
    name = models.CharField('name', max_length=100)

    def __unicode__(self):
        return self.user.get_username()

class Field(models.Model):
    FTYPE_CHOICES = (
        ('NUMBER', 'Number'),
        ('STRING', 'String'),
        ('BOOLEAN', 'Boolean'),
    )
    fname = models.CharField(max_length=50, default='')
    ftype = models.CharField(max_length=50, choices=FTYPE_CHOICES, default=FTYPE_CHOICES[0][0])  # noqa

    def __unicode__(self):
        return '%d - %s(%s)' % (self.id, self.fname, self.ftype)

class Mapping(models.Model):
    field = models.ForeignKey(Field)
    onto = models.CharField(max_length=50, default='')

    def __unicode__(self):
        return '%d - [%s -> %s]' % (self.id, self.field.fname, self.onto)

class Template(models.Model):
    tname = models.CharField(max_length=50, default='')
    owner = models.ForeignKey(Member)
    description = models.TextField(blank=True)
    sample = models.TextField(blank=True)
    # html describe that template
    html_src = models.TextField(blank=True)
    thumbnail_src = models.FileField(upload_to='upload_img')
    # Store the struct of the template
    fields = models.ManyToManyField(Field, blank=True, null=True)

    def __unicode__(self):
        return '%d - %s' % (self.id, self.tname)

class Graph(models.Model):
    gname = models.CharField(max_length=50, default='')
    owner = models.ForeignKey(Member)
    template = models.ForeignKey(Template)
    description = models.TextField(blank=True)
    # json file
    data_src = models.TextField()
    mappings = models.ManyToManyField(Mapping, blank=True, null=True)

    def __unicode__(self):
        return '%d - %s' % (self.id, self.gname)


text:
import {selector} from "d3-selection";
import {Transition} from "./index";
import schedule, {get} from "./schedule";

export default function(select) {
  var name = this._name,
      id = this._id;

  if (typeof select !== "function") select = selector(select);

  for (var groups = this._groups, m = groups.length, subgroups = new Array(m), j = 0; j < m; ++j) {
    for (var group = groups[j], n = group.length, subgroup = subgroups[j] = new Array(n), node, subnode, i = 0; i < n; ++i) {
      if ((node = group[i]) && (subnode = select.call(node, node.__data__, i, group))) {
        if ("__data__" in node) subnode.__data__ = node.__data__;
        subgroup[i] = subnode;
        schedule(subgroup[i], name, id, i, subgroup, get(node, id));
      }
    }
  }

  return new Transition(subgroups, this._parents, name, id);
}


text:
"""Helpers used for upgrading between different database formats."""

import os
import re
import shutil
import samba

from samba import Ldb, version, ntacls
from ldb import SCOPE_SUBTREE, SCOPE_ONELEVEL, SCOPE_BASE
import ldb
from samba.provision import (provision_paths_from_lp,
                            getpolicypath, set_gpos_acl, create_gpo_struct,
                            provision, ProvisioningError,
                            setsysvolacl, secretsdb_self_join)
from samba.provision.common import FILL_FULL
from samba.dcerpc import xattr, drsblobs, security
from samba.dcerpc.misc import SEC_CHAN_BDC
from samba.ndr import ndr_unpack
from samba.samdb import SamDB
from samba import _glue
import tempfile

# All the ldb related to registry are commented because the path for them is
# relative in the provisionPath object
# And so opening them create a file in the current directory which is not what
# we want
# I still keep them commented because I plan soon to make more cleaner
ERROR =     -1
SIMPLE =     0x00
CHANGE =     0x01
CHANGESD =     0x02
GUESS =     0x04
PROVISION =    0x08
CHANGEALL =    0xff

hashAttrNotCopied = set(["dn", "whenCreated", "whenChanged", "objectGUID",
    "uSNCreated", "replPropertyMetaData", "uSNChanged", "parentGUID",
    "objectCategory", "distinguishedName", "nTMixedDomain",
    "showInAdvancedViewOnly", "instanceType", "msDS-Behavior-Version",
    "nextRid", "cn", "versionNumber", "lmPwdHistory", "pwdLastSet",
    "ntPwdHistory", "unicodePwd","dBCSPwd", "supplementalCredentials",
    "gPCUserExtensionNames", "gPCMachineExtensionNames","maxPwdAge", "secret",
    "possibleInferiors", "privilege", "sAMAccountType"])

class ProvisionLDB(object):

    def __init__(self):
        self.sam = None
        self.secrets = None
        self.idmap = None
        self.privilege = None
        self.hkcr = None
        self.hkcu = None
        self.hku = None
        self.hklm = None

    def dbs(self):
        return (self.sam, self.secrets, self.idmap, self.privilege)

    def startTransactions(self):
        for db in self.dbs():
            db.transaction_start()
# TO BE DONE
#        self.hkcr.transaction_start()
#        self.hkcu.transaction_start()
#        self.hku.transaction_start()
#        self.hklm.transaction_start()

    def groupedRollback(self):
        ok = True
        for db in self.dbs():
            try:
                db.transaction_cancel()
            except Exception:
                ok = False
        return ok
# TO BE DONE
#        self.hkcr.transaction_cancel()
#        self.hkcu.transaction_cancel()
#        self.hku.transaction_cancel()
#        self.hklm.transaction_cancel()

    def groupedCommit(self):
        try:
            for db in self.dbs():
                db.transaction_prepare_commit()
        except Exception:
            return self.groupedRollback()
# TO BE DONE
#        self.hkcr.transaction_prepare_commit()
#        self.hkcu.transaction_prepare_commit()
#        self.hku.transaction_prepare_commit()
#        self.hklm.transaction_prepare_commit()
        try:
            for db in self.dbs():
                db.transaction_commit()
        except Exception:
            return self.groupedRollback()

# TO BE DONE
#        self.hkcr.transaction_commit()
#        self.hkcu.transaction_commit()
#        self.hku.transaction_commit()
#        self.hklm.transaction_commit()
        return True

def get_ldbs(paths, creds, session, lp):
    """Return LDB object mapped on most important databases

    :param paths: An object holding the different importants paths for provision object
    :param creds: Credential used for openning LDB files
    :param session: Session to use for openning LDB files
    :param lp: A loadparam object
    :return: A ProvisionLDB object that contains LDB object for the different LDB files of the provision"""

    ldbs = ProvisionLDB()

    ldbs.sam = SamDB(paths.samdb, session_info=session, credentials=creds, lp=lp, options=["modules:samba_dsdb"])
    ldbs.secrets = Ldb(paths.secrets, session_info=session, credentials=creds, lp=lp)
    ldbs.idmap = Ldb(paths.idmapdb, session_info=session, credentials=creds, lp=lp)
    ldbs.privilege = Ldb(paths.privilege, session_info=session, credentials=creds, lp=lp)
#    ldbs.hkcr = Ldb(paths.hkcr, session_info=session, credentials=creds, lp=lp)
#    ldbs.hkcu = Ldb(paths.hkcu, session_info=session, credentials=creds, lp=lp)
#    ldbs.hku = Ldb(paths.hku, session_info=session, credentials=creds, lp=lp)
#    ldbs.hklm = Ldb(paths.hklm, session_info=session, credentials=creds, lp=lp)

    return ldbs

def usn_in_range(usn, range):
    """Check if the usn is in one of the range provided.
    To do so, the value is checked to be between the lower bound and
    higher bound of a range

    :param usn: A integer value corresponding to the usn that we want to update
    :param range: A list of integer representing ranges, lower bounds are in
                  the even indices, higher in odd indices
    :return: True if the usn is in one of the range, False otherwise
    """

    idx = 0
    cont = True
    ok = False
    while cont:
        if idx ==  len(range):
            cont = False
            continue
        if usn < int(range[idx]):
            if idx %2 == 1:
                ok = True
            cont = False
        if usn == int(range[idx]):
            cont = False
            ok = True
        idx = idx + 1
    return ok

def get_paths(param, targetdir=None, smbconf=None):
    """Get paths to important provision objects (smb.conf, ldb files, ...)

    :param param: Param object
    :param targetdir: Directory where the provision is (or will be) stored
    :param smbconf: Path to the smb.conf file
    :return: A list with the path of important provision objects"""
    if targetdir is not None:
        if not os.path.exists(targetdir):
            os.mkdir(targetdir)
        etcdir = os.path.join(targetdir, "etc")
        if not os.path.exists(etcdir):
            os.makedirs(etcdir)
        smbconf = os.path.join(etcdir, "smb.conf")
    if smbconf is None:
        smbconf = param.default_path()

    if not os.path.exists(smbconf):
        raise ProvisioningError("Unable to find smb.conf")

    lp = param.LoadParm()
    lp.load(smbconf)
    paths = provision_paths_from_lp(lp, lp.get("realm"))
    return paths

def update_policyids(names, samdb):
    """Update policy ids that could have changed after sam update

    :param names: List of key provision parameters
    :param samdb: An Ldb object conntected with the sam DB
    """
    # policy guid
    res = samdb.search(expression="(displayName=Default Domain Policy)",
                        base="CN=Policies,CN=System," + str(names.rootdn),
                        scope=SCOPE_ONELEVEL, attrs=["cn","displayName"])
    names.policyid = str(res[0]["cn"]).replace("{","").replace("}","")
    # dc policy guid
    res2 = samdb.search(expression="(displayName=Default Domain Controllers"
                                   " Policy)",
                            base="CN=Policies,CN=System," + str(names.rootdn),
                            scope=SCOPE_ONELEVEL, attrs=["cn","displayName"])
    if len(res2) == 1:
        names.policyid_dc = str(res2[0]["cn"]).replace("{","").replace("}","")
    else:
        names.policyid_dc = None

def newprovision(names, session, smbconf, provdir, logger):
    """Create a new provision.

    This provision will be the reference for knowing what has changed in the
    since the latest upgrade in the current provision

    :param names: List of provision parameters
    :param creds: Credentials for the authentification
    :param session: Session object
    :param smbconf: Path to the smb.conf file
    :param provdir: Directory where the provision will be stored
    :param logger: A Logger
    """
    if os.path.isdir(provdir):
        shutil.rmtree(provdir)
    os.mkdir(provdir)
    logger.info("Provision stored in %s", provdir)
    return provision(logger, session, smbconf=smbconf,
            targetdir=provdir, samdb_fill=FILL_FULL, realm=names.realm,
            domain=names.domain, domainguid=names.domainguid,
            domainsid=names.domainsid, ntdsguid=names.ntdsguid,
            policyguid=names.policyid, policyguid_dc=names.policyid_dc,
            hostname=names.netbiosname.lower(), hostip=None, hostip6=None,
            invocationid=names.invocation, adminpass=names.adminpass,
            krbtgtpass=None, machinepass=None, dnspass=None, root=None,
            nobody=None, users=None,
            serverrole="domain controller",
            backend_type=None, ldapadminpass=None, ol_mmr_urls=None,
            slapd_path=None,
            dom_for_fun_level=names.domainlevel, dns_backend=names.dns_backend,
            useeadb=True, use_ntvfs=True)

def dn_sort(x, y):
    """Sorts two DNs in the lexicographical order it and put higher level DN
    before.

    So given the dns cn=bar,cn=foo and cn=foo the later will be return as
    smaller

    :param x: First object to compare
    :param y: Second object to compare
    """
    p = re.compile(r'(?<!\\), ?')
    tab1 = p.split(str(x))
    tab2 = p.split(str(y))
    minimum = min(len(tab1), len(tab2))
    len1 = len(tab1)-1
    len2 = len(tab2)-1
    # Note: python range go up to upper limit but do not include it
    for i in range(0, minimum):
        ret = cmp(tab1[len1-i], tab2[len2-i])
        if ret != 0:
            return ret
        else:
            if i == minimum-1:
                assert len1!=len2,"PB PB PB" + " ".join(tab1)+" / " + " ".join(tab2)
                if len1 > len2:
                    return 1
                else:
                    return -1
    return ret

def identic_rename(ldbobj, dn):
    """Perform a back and forth rename to trigger renaming on attribute that
    can't be directly modified.

    :param lbdobj: An Ldb Object
    :param dn: DN of the object to manipulate
    """
    (before, after) = str(dn).split('=', 1)
    # we need to use relax to avoid the subtree_rename constraints
    ldbobj.rename(dn, ldb.Dn(ldbobj, "%s=foo%s" % (before, after)), ["relax:0"])
    ldbobj.rename(ldb.Dn(ldbobj, "%s=foo%s" % (before, after)), dn, ["relax:0"])

def update_secrets(newsecrets_ldb, secrets_ldb, messagefunc):
    """Update secrets.ldb

    :param newsecrets_ldb: An LDB object that is connected to the secrets.ldb
        of the reference provision
    :param secrets_ldb: An LDB object that is connected to the secrets.ldb
        of the updated provision
    """

    messagefunc(SIMPLE, "Update of secrets.ldb")
    reference = newsecrets_ldb.search(base="@MODULES", scope=SCOPE_BASE)
    current = secrets_ldb.search(base="@MODULES", scope=SCOPE_BASE)
    assert reference, "Reference modules list can not be empty"
    if len(current) == 0:
        # No modules present
        delta = secrets_ldb.msg_diff(ldb.Message(), reference[0])
        delta.dn = reference[0].dn
        secrets_ldb.add(reference[0])
    else:
        delta = secrets_ldb.msg_diff(current[0], reference[0])
        delta.dn = current[0].dn
        secrets_ldb.modify(delta)

    reference = newsecrets_ldb.search(expression="objectClass=top", base="",
                                        scope=SCOPE_SUBTREE, attrs=["dn"])
    current = secrets_ldb.search(expression="objectClass=top", base="",
                                        scope=SCOPE_SUBTREE, attrs=["dn"])
    hash_new = {}
    hash = {}
    listMissing = []
    listPresent = []

    empty = ldb.Message()
    for i in range(0, len(reference)):
        hash_new[str(reference[i]["dn"]).lower()] = reference[i]["dn"]

    # Create a hash for speeding the search of existing object in the
    # current provision
    for i in range(0, len(current)):
        hash[str(current[i]["dn"]).lower()] = current[i]["dn"]

    for k in hash_new.keys():
        if not hash.has_key(k):
            listMissing.append(hash_new[k])
        else:
            listPresent.append(hash_new[k])

    for entry in listMissing:
        reference = newsecrets_ldb.search(expression="distinguishedName=%s" % entry,
                                            base="", scope=SCOPE_SUBTREE)
        current = secrets_ldb.search(expression="distinguishedName=%s" % entry,
                                            base="", scope=SCOPE_SUBTREE)
        delta = secrets_ldb.msg_diff(empty, reference[0])
        for att in hashAttrNotCopied:
            delta.remove(att)
        messagefunc(CHANGE, "Entry %s is missing from secrets.ldb" %
                    reference[0].dn)
        for att in delta:
            messagefunc(CHANGE, " Adding attribute %s" % att)
        delta.dn = reference[0].dn
        secrets_ldb.add(delta)

    for entry in listPresent:
        reference = newsecrets_ldb.search(expression="distinguishedName=%s" % entry,
                                            base="", scope=SCOPE_SUBTREE)
        current = secrets_ldb.search(expression="distinguishedName=%s" % entry, base="",
                                            scope=SCOPE_SUBTREE)
        delta = secrets_ldb.msg_diff(current[0], reference[0])
        for att in hashAttrNotCopied:
            delta.remove(att)
        for att in delta:
            if att == "name":
                messagefunc(CHANGE, "Found attribute name on  %s,"
                                    " must rename the DN" % (current[0].dn))
                identic_rename(secrets_ldb, reference[0].dn)
            else:
                delta.remove(att)

    for entry in listPresent:
        reference = newsecrets_ldb.search(expression="distinguishedName=%s" % entry, base="",
                                            scope=SCOPE_SUBTREE)
        current = secrets_ldb.search(expression="distinguishedName=%s" % entry, base="",
                                            scope=SCOPE_SUBTREE)
        delta = secrets_ldb.msg_diff(current[0], reference[0])
        for att in hashAttrNotCopied:
            delta.remove(att)
        for att in delta:
            if att == "msDS-KeyVersionNumber":
                delta.remove(att)
            if att != "dn":
                messagefunc(CHANGE,
                            "Adding/Changing attribute %s to %s" %
                            (att, current[0].dn))

        delta.dn = current[0].dn
        secrets_ldb.modify(delta)

    res2 = secrets_ldb.search(expression="(samaccountname=dns)",
                                scope=SCOPE_SUBTREE, attrs=["dn"])

    if len(res2) == 1:
            messagefunc(SIMPLE, "Remove old dns account")
            secrets_ldb.delete(res2[0]["dn"])

def getOEMInfo(samdb, rootdn):
    """Return OEM Information on the top level Samba4 use to store version
    info in this field

    :param samdb: An LDB object connect to sam.ldb
    :param rootdn: Root DN of the domain
    :return: The content of the field oEMInformation (if any)
    """
    res = samdb.search(expression="(objectClass=*)", base=str(rootdn),
                            scope=SCOPE_BASE, attrs=["dn", "oEMInformation"])
    if len(res) > 0 and res[0].get("oEMInformation"):
        info = res[0]["oEMInformation"]
        return info
    else:
        return ""

def updateOEMInfo(samdb, rootdn):
    """Update the OEMinfo field to add information about upgrade

    :param samdb: an LDB object connected to the sam DB
    :param rootdn: The string representation of the root DN of
        the provision (ie. DC=...,DC=...)
    """
    res = samdb.search(expression="(objectClass=*)", base=rootdn,
                            scope=SCOPE_BASE, attrs=["dn", "oEMInformation"])
    if len(res) > 0:
        if res[0].get("oEMInformation"):
            info = str(res[0]["oEMInformation"])
        else:
            info = ""
        info = "%s, upgrade to %s" % (info, version)
        delta = ldb.Message()
        delta.dn = ldb.Dn(samdb, str(res[0]["dn"]))
        delta["oEMInformation"] = ldb.MessageElement(info, ldb.FLAG_MOD_REPLACE,
                                                        "oEMInformation" )
        samdb.modify(delta)

def update_gpo(paths, samdb, names, lp, message):
    """Create missing GPO file object if needed
    """
    dir = getpolicypath(paths.sysvol, names.dnsdomain, names.policyid)
    if not os.path.isdir(dir):
        create_gpo_struct(dir)

    if names.policyid_dc is None:
        raise ProvisioningError("Policy ID for Domain controller is missing")
    dir = getpolicypath(paths.sysvol, names.dnsdomain, names.policyid_dc)
    if not os.path.isdir(dir):
        create_gpo_struct(dir)

def increment_calculated_keyversion_number(samdb, rootdn, hashDns):
    """For a given hash associating dn and a number, this function will
    update the replPropertyMetaData of each dn in the hash, so that the
    calculated value of the msDs-KeyVersionNumber is equal or superior to the
    one associated to the given dn.

    :param samdb: An SamDB object pointing to the sam
    :param rootdn: The base DN where we want to start
    :param hashDns: A hash with dn as key and number representing the
                 minimum value of msDs-KeyVersionNumber that we want to
                 have
    """
    entry = samdb.search(expression='(objectClass=user)',
                         base=ldb.Dn(samdb,str(rootdn)),
                         scope=SCOPE_SUBTREE, attrs=["msDs-KeyVersionNumber"],
                         controls=["search_options:1:2"])
    done = 0
    hashDone = {}
    if len(entry) == 0:
        raise ProvisioningError("Unable to find msDs-KeyVersionNumber")
    else:
        for e in entry:
            if hashDns.has_key(str(e.dn).lower()):
                val = e.get("msDs-KeyVersionNumber")
                if not val:
                    val = "0"
                version = int(str(hashDns[str(e.dn).lower()]))
                if int(str(val)) < version:
                    done = done + 1
                    samdb.set_attribute_replmetadata_version(str(e.dn),
                                                              "unicodePwd",
                                                              version, True)
def delta_update_basesamdb(refsampath, sampath, creds, session, lp, message):
    """Update the provision container db: sam.ldb
    This function is aimed for alpha9 and newer;

    :param refsampath: Path to the samdb in the reference provision
    :param sampath: Path to the samdb in the upgraded provision
    :param creds: Credential used for openning LDB files
    :param session: Session to use for openning LDB files
    :param lp: A loadparam object
    :return: A msg_diff object with the difference between the @ATTRIBUTES
             of the current provision and the reference provision
    """

    message(SIMPLE,
            "Update base samdb by searching difference with reference one")
    refsam = Ldb(refsampath, session_info=session, credentials=creds,
                    lp=lp, options=["modules:"])
    sam = Ldb(sampath, session_info=session, credentials=creds, lp=lp,
                options=["modules:"])

    empty = ldb.Message()
    deltaattr = None
    reference = refsam.search(expression="")

    for refentry in reference:
        entry = sam.search(expression="distinguishedName=%s" % refentry["dn"],
                            scope=SCOPE_SUBTREE)
        if not len(entry):
            delta = sam.msg_diff(empty, refentry)
            message(CHANGE, "Adding %s to sam db" % str(refentry.dn))
            if str(refentry.dn) == "@PROVISION" and\
                delta.get(samba.provision.LAST_PROVISION_USN_ATTRIBUTE):
                delta.remove(samba.provision.LAST_PROVISION_USN_ATTRIBUTE)
            delta.dn = refentry.dn
            sam.add(delta)
        else:
            delta = sam.msg_diff(entry[0], refentry)
            if str(refentry.dn) == "@ATTRIBUTES":
                deltaattr = sam.msg_diff(refentry, entry[0])
            if str(refentry.dn) == "@PROVISION" and\
                delta.get(samba.provision.LAST_PROVISION_USN_ATTRIBUTE):
                delta.remove(samba.provision.LAST_PROVISION_USN_ATTRIBUTE)
            if len(delta.items()) > 1:
                delta.dn = refentry.dn
                sam.modify(delta)

    return deltaattr

def construct_existor_expr(attrs):
    """Construct a exists or LDAP search expression.

    :param attrs: List of attribute on which we want to create the search
        expression.
    :return: A string representing the expression, if attrs is empty an
        empty string is returned
    """
    expr = ""
    if len(attrs) > 0:
        expr = "(|"
        for att in attrs:
            expr = "%s(%s=*)"%(expr,att)
        expr = "%s)"%expr
    return expr

def update_machine_account_password(samdb, secrets_ldb, names):
    """Update (change) the password of the current DC both in the SAM db and in
       secret one

    :param samdb: An LDB object related to the sam.ldb file of a given provision
    :param secrets_ldb: An LDB object related to the secrets.ldb file of a given
                        provision
    :param names: List of key provision parameters"""

    expression = "samAccountName=%s$" % names.netbiosname
    secrets_msg = secrets_ldb.search(expression=expression,
                                        attrs=["secureChannelType"])
    if int(secrets_msg[0]["secureChannelType"][0]) == SEC_CHAN_BDC:
        res = samdb.search(expression=expression, attrs=[])
        assert(len(res) == 1)

        msg = ldb.Message(res[0].dn)
        machinepass = samba.generate_random_password(128, 255)
        mputf16 = machinepass.encode('utf-16-le')
        msg["clearTextPassword"] = ldb.MessageElement(mputf16,
                                                ldb.FLAG_MOD_REPLACE,
                                                "clearTextPassword")
        samdb.modify(msg)

        res = samdb.search(expression=("samAccountName=%s$" % names.netbiosname),
                     attrs=["msDs-keyVersionNumber"])
        assert(len(res) == 1)
        kvno = int(str(res[0]["msDs-keyVersionNumber"]))
        secChanType = int(secrets_msg[0]["secureChannelType"][0])

        secretsdb_self_join(secrets_ldb, domain=names.domain,
                    realm=names.realm,
                    domainsid=names.domainsid,
                    dnsdomain=names.dnsdomain,
                    netbiosname=names.netbiosname,
                    machinepass=machinepass,
                    key_version_number=kvno,
                    secure_channel_type=secChanType)
    else:
        raise ProvisioningError("Unable to find a Secure Channel"
                                "of type SEC_CHAN_BDC")

def update_dns_account_password(samdb, secrets_ldb, names):
    """Update (change) the password of the dns both in the SAM db and in
       secret one

    :param samdb: An LDB object related to the sam.ldb file of a given provision
    :param secrets_ldb: An LDB object related to the secrets.ldb file of a given
                        provision
    :param names: List of key provision parameters"""

    expression = "samAccountName=dns-%s" % names.netbiosname
    secrets_msg = secrets_ldb.search(expression=expression)
    if len(secrets_msg) == 1:
        res = samdb.search(expression=expression, attrs=[])
        assert(len(res) == 1)

        msg = ldb.Message(res[0].dn)
        machinepass = samba.generate_random_password(128, 255)
        mputf16 = machinepass.encode('utf-16-le')
        msg["clearTextPassword"] = ldb.MessageElement(mputf16,
                                                ldb.FLAG_MOD_REPLACE,
                                                "clearTextPassword")

        samdb.modify(msg)

        res = samdb.search(expression=expression,
                     attrs=["msDs-keyVersionNumber"])
        assert(len(res) == 1)
        kvno = str(res[0]["msDs-keyVersionNumber"])

        msg = ldb.Message(secrets_msg[0].dn)
        msg["secret"] = ldb.MessageElement(machinepass,
                                                ldb.FLAG_MOD_REPLACE,
                                                "secret")
        msg["msDS-KeyVersionNumber"] = ldb.MessageElement(kvno,
                                                ldb.FLAG_MOD_REPLACE,
                                                "msDS-KeyVersionNumber")

        secrets_ldb.modify(msg)

def update_krbtgt_account_password(samdb, names):
    """Update (change) the password of the krbtgt account

    :param samdb: An LDB object related to the sam.ldb file of a given provision
    :param names: List of key provision parameters"""

    expression = "samAccountName=krbtgt"
    res = samdb.search(expression=expression, attrs=[])
    assert(len(res) == 1)

    msg = ldb.Message(res[0].dn)
    machinepass = samba.generate_random_password(128, 255)
    mputf16 = machinepass.encode('utf-16-le')
    msg["clearTextPassword"] = ldb.MessageElement(mputf16,
                                                  ldb.FLAG_MOD_REPLACE,
                                                  "clearTextPassword")

    samdb.modify(msg)

def search_constructed_attrs_stored(samdb, rootdn, attrs):
    """Search a given sam DB for calculated attributes that are
    still stored in the db.

    :param samdb: An LDB object pointing to the sam
    :param rootdn: The base DN where the search should start
    :param attrs: A list of attributes to be searched
    :return: A hash with attributes as key and an array of
             array. Each array contains the dn and the associated
             values for this attribute as they are stored in the
             sam."""

    hashAtt = {}
    expr = construct_existor_expr(attrs)
    if expr == "":
        return hashAtt
    entry = samdb.search(expression=expr, base=ldb.Dn(samdb, str(rootdn)),
                         scope=SCOPE_SUBTREE, attrs=attrs,
                         controls=["search_options:1:2","bypassoperational:0"])
    if len(entry) == 0:
        # Nothing anymore
        return hashAtt

    for ent in entry:
        for att in attrs:
            if ent.get(att):
                if hashAtt.has_key(att):
                    hashAtt[att][str(ent.dn).lower()] = str(ent[att])
                else:
                    hashAtt[att] = {}
                    hashAtt[att][str(ent.dn).lower()] = str(ent[att])

    return hashAtt

def findprovisionrange(samdb, basedn):
    """ Find ranges of usn grouped by invocation id and then by timestamp
        rouned at 1 minute

        :param samdb: An LDB object pointing to the samdb
        :param basedn: The DN of the forest

        :return: A two level dictionary with invoication id as the
                first level, timestamp as the second one and then
                max, min, and number as subkeys, representing respectivily
                the maximum usn for the range, the minimum usn and the number
                of object with usn in this range.
    """
    nb_obj = 0
    hash_id = {}

    res = samdb.search(base=basedn, expression="objectClass=*",
                                    scope=ldb.SCOPE_SUBTREE,
                                    attrs=["replPropertyMetaData"],
                                    controls=["search_options:1:2"])

    for e in res:
        nb_obj = nb_obj + 1
        obj = ndr_unpack(drsblobs.replPropertyMetaDataBlob,
                            str(e["replPropertyMetaData"])).ctr

        for o in obj.array:
            # like a timestamp but with the resolution of 1 minute
            minutestamp =_glue.nttime2unix(o.originating_change_time)/60
            hash_ts = hash_id.get(str(o.originating_invocation_id))

            if hash_ts is None:
                ob = {}
                ob["min"] = o.originating_usn
                ob["max"] = o.originating_usn
                ob["num"] = 1
                ob["list"] = [str(e.dn)]
                hash_ts = {}
            else:
                ob = hash_ts.get(minutestamp)
                if ob is None:
                    ob = {}
                    ob["min"] = o.originating_usn
                    ob["max"] = o.originating_usn
                    ob["num"] = 1
                    ob["list"] = [str(e.dn)]
                else:
                    if ob["min"] > o.originating_usn:
                        ob["min"] = o.originating_usn
                    if ob["max"] < o.originating_usn:
                        ob["max"] = o.originating_usn
                    if not (str(e.dn) in ob["list"]):
                        ob["num"] = ob["num"] + 1
                        ob["list"].append(str(e.dn))
            hash_ts[minutestamp] = ob
            hash_id[str(o.originating_invocation_id)] = hash_ts

    return (hash_id, nb_obj)

def print_provision_ranges(dic, limit_print, dest, samdb_path, invocationid):
    """ print the differents ranges passed as parameter

        :param dic: A dictionnary as returned by findprovisionrange
        :param limit_print: minimum number of object in a range in order to print it
        :param dest: Destination directory
        :param samdb_path: Path to the sam.ldb file
        :param invoicationid: Invocation ID for the current provision
    """
    ldif = ""

    for id in dic:
        hash_ts = dic[id]
        sorted_keys = []
        sorted_keys.extend(hash_ts.keys())
        sorted_keys.sort()

        kept_record = []
        for k in sorted_keys:
            obj = hash_ts[k]
            if obj["num"] > limit_print:
                dt = _glue.nttime2string(_glue.unix2nttime(k*60))
                print "%s # of modification: %d  \tmin: %d max: %d" % (dt , obj["num"],
                                                                    obj["min"],
                                                                    obj["max"])
            if hash_ts[k]["num"] > 600:
                kept_record.append(k)

        # Let's try to concatenate consecutive block if they are in the almost same minutestamp
        for i in range(0, len(kept_record)):
            if i != 0:
                key1 = kept_record[i]
                key2 = kept_record[i-1]
                if key1 - key2 == 1:
                    # previous record is just 1 minute away from current
                    if int(hash_ts[key1]["min"]) == int(hash_ts[key2]["max"]) + 1:
                        # Copy the highest USN in the previous record
                        # and mark the current as skipped
                        hash_ts[key2]["max"] = hash_ts[key1]["max"]
                        hash_ts[key1]["skipped"] = True

        for k in kept_record:
                obj = hash_ts[k]
                if obj.get("skipped") is None:
                    ldif = "%slastProvisionUSN: %d-%d;%s\n" % (ldif, obj["min"],
                                obj["max"], id)

    if ldif != "":
        file = tempfile.mktemp(dir=dest, prefix="usnprov", suffix=".ldif")
        print
        print "To track the USNs modified/created by provision and upgrade proivsion,"
        print " the following ranges are proposed to be added to your provision sam.ldb: \n%s" % ldif
        print "We recommend to review them, and if it's correct to integrate the following ldif: %s in your sam.ldb" % file
        print "You can load this file like this: ldbadd -H %s %s\n"%(str(samdb_path),file)
        ldif = "dn: @PROVISION\nprovisionnerID: %s\n%s" % (invocationid, ldif)
        open(file,'w').write(ldif)

def int64range2str(value):
    """Display the int64 range stored in value as xxx-yyy

    :param value: The int64 range
    :return: A string of the representation of the range
    """

    lvalue = long(value)
    str = "%d-%d" % (lvalue&0xFFFFFFFF, lvalue>>32)
    return str


text:
package org.nd4j.parameterserver.distributed.conf;

import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
import org.nd4j.BaseND4JTest;
import org.nd4j.linalg.exception.ND4JIllegalStateException;

import static org.junit.Assert.*;

/**
 * @author raver119@gmail.com
 */
public class VoidConfigurationTest extends BaseND4JTest {

    @Rule
    public Timeout globalTimeout = Timeout.seconds(30);

    @Test
    public void testNetworkMask1() throws Exception {
        VoidConfiguration configuration = new VoidConfiguration();
        configuration.setNetworkMask("192.168.1.0/24");

        assertEquals("192.168.1.0/24", configuration.getNetworkMask());
    }

    @Test
    public void testNetworkMask2() throws Exception {
        VoidConfiguration configuration = new VoidConfiguration();
        configuration.setNetworkMask("192.168.1.12");

        assertEquals("192.168.1.0/24", configuration.getNetworkMask());
    }

    @Test
    public void testNetworkMask5() throws Exception {
        VoidConfiguration configuration = new VoidConfiguration();
        configuration.setNetworkMask("192.168.0.0/16");

        assertEquals("192.168.0.0/16", configuration.getNetworkMask());
    }

    @Test
    public void testNetworkMask6() throws Exception {
        VoidConfiguration configuration = new VoidConfiguration();
        configuration.setNetworkMask("192.168.0.0/8");

        assertEquals("192.168.0.0/8", configuration.getNetworkMask());
    }

    @Test(expected = ND4JIllegalStateException.class)
    public void testNetworkMask3() throws Exception {
        VoidConfiguration configuration = new VoidConfiguration();
        configuration.setNetworkMask("192.256.1.1/24");

        assertEquals("192.168.1.0/24", configuration.getNetworkMask());
    }

    @Test(expected = ND4JIllegalStateException.class)
    public void testNetworkMask4() throws Exception {
        VoidConfiguration configuration = new VoidConfiguration();
        configuration.setNetworkMask("0.0.0.0/8");

        assertEquals("192.168.1.0/24", configuration.getNetworkMask());
    }
}


text:
package notifications

import (
	"io"
	"reflect"
	"testing"

	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/digest"
	"github.com/docker/distribution/manifest"
	"github.com/docker/distribution/manifest/schema1"
	"github.com/docker/distribution/registry/storage"
	"github.com/docker/distribution/registry/storage/cache/memory"
	"github.com/docker/distribution/registry/storage/driver/inmemory"
	"github.com/docker/distribution/testutil"
	"github.com/docker/libtrust"
)

func TestListener(t *testing.T) {
	ctx := context.Background()
	registry, err := storage.NewRegistry(ctx, inmemory.New(), storage.BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), storage.EnableDelete, storage.EnableRedirect)
	if err != nil {
		t.Fatalf("error creating registry: %v", err)
	}
	tl := &testListener{
		ops: make(map[string]int),
	}

	repository, err := registry.Repository(ctx, "foo/bar")
	if err != nil {
		t.Fatalf("unexpected error getting repo: %v", err)
	}
	repository = Listen(repository, tl)

	// Now take the registry through a number of operations
	checkExerciseRepository(t, repository)

	expectedOps := map[string]int{
		"manifest:push": 1,
		"manifest:pull": 2,
		// "manifest:delete": 0, // deletes not supported for now
		"layer:push": 2,
		"layer:pull": 2,
		// "layer:delete":    0, // deletes not supported for now
	}

	if !reflect.DeepEqual(tl.ops, expectedOps) {
		t.Fatalf("counts do not match:\n%v\n !=\n%v", tl.ops, expectedOps)
	}

}

type testListener struct {
	ops map[string]int
}

func (tl *testListener) ManifestPushed(repo string, sm *schema1.SignedManifest) error {
	tl.ops["manifest:push"]++

	return nil
}

func (tl *testListener) ManifestPulled(repo string, sm *schema1.SignedManifest) error {
	tl.ops["manifest:pull"]++
	return nil
}

func (tl *testListener) ManifestDeleted(repo string, sm *schema1.SignedManifest) error {
	tl.ops["manifest:delete"]++
	return nil
}

func (tl *testListener) BlobPushed(repo string, desc distribution.Descriptor) error {
	tl.ops["layer:push"]++
	return nil
}

func (tl *testListener) BlobPulled(repo string, desc distribution.Descriptor) error {
	tl.ops["layer:pull"]++
	return nil
}

func (tl *testListener) BlobDeleted(repo string, desc distribution.Descriptor) error {
	tl.ops["layer:delete"]++
	return nil
}

// checkExerciseRegistry takes the registry through all of its operations,
// carrying out generic checks.
func checkExerciseRepository(t *testing.T, repository distribution.Repository) {
	// TODO(stevvooe): This would be a nice testutil function. Basically, it
	// takes the registry through a common set of operations. This could be
	// used to make cross-cutting updates by changing internals that affect
	// update counts. Basically, it would make writing tests a lot easier.
	ctx := context.Background()
	tag := "thetag"
	m := schema1.Manifest{
		Versioned: manifest.Versioned{
			SchemaVersion: 1,
		},
		Name: repository.Name(),
		Tag:  tag,
	}

	blobs := repository.Blobs(ctx)
	for i := 0; i < 2; i++ {
		rs, ds, err := testutil.CreateRandomTarFile()
		if err != nil {
			t.Fatalf("error creating test layer: %v", err)
		}
		dgst := digest.Digest(ds)

		wr, err := blobs.Create(ctx)
		if err != nil {
			t.Fatalf("error creating layer upload: %v", err)
		}

		// Use the resumes, as well!
		wr, err = blobs.Resume(ctx, wr.ID())
		if err != nil {
			t.Fatalf("error resuming layer upload: %v", err)
		}

		io.Copy(wr, rs)

		if _, err := wr.Commit(ctx, distribution.Descriptor{Digest: dgst}); err != nil {
			t.Fatalf("unexpected error finishing upload: %v", err)
		}

		m.FSLayers = append(m.FSLayers, schema1.FSLayer{
			BlobSum: dgst,
		})
		m.History = append(m.History, schema1.History{
			V1Compatibility: "",
		})

		// Then fetch the blobs
		if rc, err := blobs.Open(ctx, dgst); err != nil {
			t.Fatalf("error fetching layer: %v", err)
		} else {
			defer rc.Close()
		}
	}

	pk, err := libtrust.GenerateECP256PrivateKey()
	if err != nil {
		t.Fatalf("unexpected error generating key: %v", err)
	}

	sm, err := schema1.Sign(&m, pk)
	if err != nil {
		t.Fatalf("unexpected error signing manifest: %v", err)
	}

	manifests, err := repository.Manifests(ctx)
	if err != nil {
		t.Fatal(err.Error())
	}

	if err = manifests.Put(sm); err != nil {
		t.Fatalf("unexpected error putting the manifest: %v", err)
	}

	p, err := sm.Payload()
	if err != nil {
		t.Fatalf("unexpected error getting manifest payload: %v", err)
	}

	dgst := digest.FromBytes(p)
	fetchedByManifest, err := manifests.Get(dgst)
	if err != nil {
		t.Fatalf("unexpected error fetching manifest: %v", err)
	}

	if fetchedByManifest.Tag != sm.Tag {
		t.Fatalf("retrieved unexpected manifest: %v", err)
	}

	fetched, err := manifests.GetByTag(tag)
	if err != nil {
		t.Fatalf("unexpected error fetching manifest: %v", err)
	}

	if fetched.Tag != fetchedByManifest.Tag {
		t.Fatalf("retrieved unexpected manifest: %v", err)
	}
}


text:
package org.keycloak.dom.saml.v2.assertion;

import java.io.Serializable;

/**
 *

 * Java class for SubjectConfirmationType complex type.
 *
 *

 * The following schema fragment specifies the expected content contained within this class.
 *
 *

 * <complexType name="SubjectConfirmationType">
 *   <complexContent>
 *     <restriction base="{http://www.w3.org/2001/XMLSchema}anyType">
 *       <sequence>
 *         <choice minOccurs="0">
 *           <element ref="{urn:oasis:names:tc:SAML:2.0:assertion}BaseID"/>
 *           <element ref="{urn:oasis:names:tc:SAML:2.0:assertion}NameID"/>
 *           <element ref="{urn:oasis:names:tc:SAML:2.0:assertion}EncryptedID"/>
 *         </choice>
 *         <element ref="{urn:oasis:names:tc:SAML:2.0:assertion}SubjectConfirmationData" minOccurs="0"/>
 *       </sequence>
 *       <attribute name="Method" use="required" type="{http://www.w3.org/2001/XMLSchema}anyURI" />
 *     </restriction>
 *   </complexContent>
 * </complexType>
 *

 */
public class SubjectConfirmationType implements Serializable {

    protected BaseIDAbstractType baseID;
    protected NameIDType nameID;
    protected EncryptedElementType encryptedID;
    protected SubjectConfirmationDataType subjectConfirmationData;
    protected String method;

    /**
     * Gets the value of the baseID property.
     *
     * @return possible object is {@link BaseIDAbstractType }
     */
    public BaseIDAbstractType getBaseID() {
        return baseID;
    }

    /**
     * Sets the value of the baseID property.
     *
     * @param value allowed object is {@link BaseIDAbstractType }
     */
    public void setBaseID(BaseIDAbstractType value) {
        this.baseID = value;
    }

    /**
     * Gets the value of the nameID property.
     *
     * @return possible object is {@link NameIDType }
     */
    public NameIDType getNameID() {
        return nameID;
    }

    /**
     * Sets the value of the nameID property.
     *
     * @param value allowed object is {@link NameIDType }
     */
    public void setNameID(NameIDType value) {
        this.nameID = value;
    }

    /**
     * Gets the value of the encryptedID property.
     *
     * @return possible object is {@link EncryptedElementType }
     */
    public EncryptedElementType getEncryptedID() {
        return encryptedID;
    }

    /**
     * Sets the value of the encryptedID property.
     *
     * @param value allowed object is {@link EncryptedElementType }
     */
    public void setEncryptedID(EncryptedElementType value) {
        this.encryptedID = value;
    }

    /**
     * Gets the value of the subjectConfirmationData property.
     *
     * @return possible object is {@link SubjectConfirmationDataType }
     */
    public SubjectConfirmationDataType getSubjectConfirmationData() {
        return subjectConfirmationData;
    }

    /**
     * Sets the value of the subjectConfirmationData property.
     *
     * @param value allowed object is {@link SubjectConfirmationDataType }
     */
    public void setSubjectConfirmationData(SubjectConfirmationDataType value) {
        this.subjectConfirmationData = value;
    }

    /**
     * Gets the value of the method property.
     *
     * @return possible object is {@link String }
     */
    public String getMethod() {
        return method;
    }

    /**
     * Sets the value of the method property.
     *
     * @param value allowed object is {@link String }
     */
    public void setMethod(String value) {
        this.method = value;
    }
}