text:
package org.apache.flink.api.java;

import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.InvalidProgramException;
import org.apache.flink.util.InstantiationUtil;

import org.apache.flink.shaded.asm6.org.objectweb.asm.ClassReader;
import org.apache.flink.shaded.asm6.org.objectweb.asm.ClassVisitor;
import org.apache.flink.shaded.asm6.org.objectweb.asm.MethodVisitor;
import org.apache.flink.shaded.asm6.org.objectweb.asm.Opcodes;

import org.apache.commons.lang3.ClassUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;

/**
 * The closure cleaner is a utility that tries to truncate the closure (enclosing instance)
 * of non-static inner classes (created for inline transformation functions). That makes non-static
 * inner classes in many cases serializable, where Java's default behavior renders them non-serializable
 * without good reason.
 */
@Internal
public class ClosureCleaner {

	private static final Logger LOG = LoggerFactory.getLogger(ClosureCleaner.class);

	/**
	 * Tries to clean the closure of the given object, if the object is a non-static inner
	 * class.
	 *
	 * @param func The object whose closure should be cleaned.
	 * @param level the clean up level.
	 * @param checkSerializable Flag to indicate whether serializability should be checked after
	 *                          the closure cleaning attempt.
	 *
	 * @throws InvalidProgramException Thrown, if 'checkSerializable' is true, and the object was
	 *                                 not serializable after the closure cleaning.
	 *
	 * @throws RuntimeException A RuntimeException may be thrown, if the code of the class could not
	 *                          be loaded, in order to process during the closure cleaning.
	 */
	public static void clean(Object func, ExecutionConfig.ClosureCleanerLevel level, boolean checkSerializable) {
		if (func == null) {
			return;
		}

		final Class<?> cls = func.getClass();

		if (ClassUtils.isPrimitiveOrWrapper(cls)) {
			return;
		}

		if (usesCustomSerialization(cls)) {
			return;
		}

		// First find the field name of the "this$0" field, this can
		// be "this$x" depending on the nesting
		boolean closureAccessed = false;

		for (Field f: cls.getDeclaredFields()) {
			if (f.getName().startsWith("this$")) {
				// found a closure referencing field - now try to clean
				closureAccessed |= cleanThis0(func, cls, f.getName());
			} else {
				Object fieldObject;
				try {
					f.setAccessible(true);
					fieldObject = f.get(func);
				} catch (IllegalAccessException e) {
					throw new RuntimeException(String.format("Can not access to the %s field in Class %s", f.getName(), func.getClass()));
				}

				/*
				 * we should do a deep clean when we encounter an anonymous class, inner class and local class, but should
				 * skip the class with custom serialize method.
				 *
				 * There are five kinds of classes (or interfaces):
				 * a) Top level classes
				 * b) Nested classes (static member classes)
				 * c) Inner classes (non-static member classes)
				 * d) Local classes (named classes declared within a method)
				 * e) Anonymous classes
				 */
				if (level == ExecutionConfig.ClosureCleanerLevel.RECURSIVE && needsRecursion(f, fieldObject)) {
					if (LOG.isDebugEnabled()) {
						LOG.debug("Dig to clean the {}", fieldObject.getClass().getName());
					}

					clean(fieldObject, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, true);
				}
			}
		}

		if (checkSerializable) {
			try {
				InstantiationUtil.serializeObject(func);
			}
			catch (Exception e) {
				String functionType = getSuperClassOrInterfaceName(func.getClass());

				String msg = functionType == null ?
						(func + " is not serializable.") :
						("The implementation of the " + functionType + " is not serializable.");

				if (closureAccessed) {
					msg += " The implementation accesses fields of its enclosing class, which is " +
							"a common reason for non-serializability. " +
							"A common solution is to make the function a proper (non-inner) class, or " +
							"a static inner class.";
				} else {
					msg += " The object probably contains or references non serializable fields.";
				}

				throw new InvalidProgramException(msg, e);
			}
		}
	}

	private static boolean needsRecursion(Field f, Object fo) {
		return (fo != null &&
				!Modifier.isStatic(f.getModifiers()) &&
				!Modifier.isTransient(f.getModifiers()));
	}

	private static boolean usesCustomSerialization(Class<?> cls) {
		try {
			cls.getDeclaredMethod("writeObject", ObjectOutputStream.class);
			return true;
		} catch (NoSuchMethodException ignored) {}

		return Externalizable.class.isAssignableFrom(cls);
	}

	public static void ensureSerializable(Object obj) {
		try {
			InstantiationUtil.serializeObject(obj);
		} catch (Exception e) {
			throw new InvalidProgramException("Object " + obj + " is not serializable", e);
		}
	}

	private static boolean cleanThis0(Object func, Class<?> cls, String this0Name) {

		This0AccessFinder this0Finder = new This0AccessFinder(this0Name);
		getClassReader(cls).accept(this0Finder, 0);

		final boolean accessesClosure = this0Finder.isThis0Accessed();

		if (LOG.isDebugEnabled()) {
			LOG.debug(this0Name + " is accessed: " + accessesClosure);
		}

		if (!accessesClosure) {
			Field this0;
			try {
				this0 = func.getClass().getDeclaredField(this0Name);
			} catch (NoSuchFieldException e) {
				// has no this$0, just return
				throw new RuntimeException("Could not set " + this0Name + ": " + e);
			}

			try {
				this0.setAccessible(true);
				this0.set(func, null);
			}
			catch (Exception e) {
				// should not happen, since we use setAccessible
				throw new RuntimeException("Could not set " + this0Name + " to null. " + e.getMessage(), e);
			}
		}

		return accessesClosure;
	}

	private static ClassReader getClassReader(Class<?> cls) {
		String className = cls.getName().replaceFirst("^.*\\.", "") + ".class";
		try {
			return new ClassReader(cls.getResourceAsStream(className));
		} catch (IOException e) {
			throw new RuntimeException("Could not create ClassReader: " + e.getMessage(), e);
		}
	}

	private static String getSuperClassOrInterfaceName(Class<?> cls) {
		Class<?> superclass = cls.getSuperclass();
		if (superclass.getName().startsWith("org.apache.flink")) {
			return superclass.getSimpleName();
		} else {
			for (Class<?> inFace : cls.getInterfaces()) {
				if (inFace.getName().startsWith("org.apache.flink")) {
					return inFace.getSimpleName();
				}
			}
			return null;
		}
	}
}

/**
 * This visitor walks methods and finds accesses to the field with the reference to
 * the enclosing class.
 */
class This0AccessFinder extends ClassVisitor {

	private final String this0Name;
	private boolean isThis0Accessed;

	public This0AccessFinder(String this0Name) {
		super(Opcodes.ASM6);
		this.this0Name = this0Name;
	}

	public boolean isThis0Accessed() {
		return isThis0Accessed;
	}

	@Override
	public MethodVisitor visitMethod(int access, String name, String desc, String sig, String[] exceptions) {
		return new MethodVisitor(Opcodes.ASM6) {

			@Override
			public void visitFieldInsn(int op, String owner, String name, String desc) {
				if (op == Opcodes.GETFIELD && name.equals(this0Name)) {
					isThis0Accessed = true;
				}
			}
		};
	}
}


text:
package iso20022

// Document that a user must file with an authorized servicer for each contract that involves foreign currency transactions with non residents.
type RegisteredContract5 struct {

	// Unique identification of the contract registration creation, amendment or closure request.
	OriginalContractRegistrationRequest *Max35Text `xml:"OrgnlCtrctRegnReq,omitempty"`

	// Financial institution of the issuer of the contract.
	IssuerFinancialInstitution *BranchAndFinancialInstitutionIdentification5 `xml:"IssrFI"`

	// Details of the contract being registered.
	Contract *UnderlyingContract1Choice `xml:"Ctrct"`

	// Contract balance on date of contract registration.
	ContractBalance []*ContractBalance1 `xml:"CtrctBal,omitempty"`

	// Type of the payment schedule provided in the contract.
	PaymentScheduleType *PaymentScheduleType1Choice `xml:"PmtSchdlTp,omitempty"`

	// Unique and unambiguous identification of the registered contract as assigned by the registration agent.
	RegisteredContractIdentification *DocumentIdentification29 `xml:"RegdCtrctId"`

	// Identification of a previously defined registered contract linked to the same underlying contract.
	//
	// Usage:
	// This is the identification of a previous contract registration for the same underlying contract, which was closed. In most cases, this is used  when a reporting party moves from one registration agent to another.
	PreviousRegisteredContractIdentification *DocumentIdentification22 `xml:"PrvsRegdCtrctId,omitempty"`

	// Journal of previously closed registered contracts for the same underlying contract, which were requested at the same registration agent.
	RegisteredContractJournal []*RegisteredContractJournal1 `xml:"RegdCtrctJrnl,omitempty"`

	// Details on amendments to the registered contract.
	Amendment []*RegisteredContractAmendment1 `xml:"Amdmnt,omitempty"`

	// Provides the communication method for the submission of the registered contract.
	Submission *RegisteredContractCommunication1 `xml:"Submissn"`

	// Provides the communication method for the delivery of the registered contract.
	Delivery *RegisteredContractCommunication1 `xml:"Dlvry"`

	// Amount of money the borrower pays back to the lender outside of interests and charges.
	//
	// Usage:
	// Only applicable for loan contracts.
	LoanPrincipalAmount *ActiveCurrencyAndAmount `xml:"LnPrncplAmt,omitempty"`

	// Indicates whether the dates provided are estimates or not.
	EstimatedDateIndicator *TrueFalseIndicator `xml:"EstmtdDtInd"`

	// Indicates whether loan in which both the lender and the borrower are divisions of the same corporation or not.
	//
	// Usage:
	// Only applicable for loan contracts.
	InterCompanyLoan *TrueFalseIndicator `xml:"IntrCpnyLn"`

	// Further information on the registered contract.
	AdditionalInformation *Max1025Text `xml:"AddtlInf,omitempty"`
}

func (r *RegisteredContract5) SetOriginalContractRegistrationRequest(value string) {
	r.OriginalContractRegistrationRequest = (*Max35Text)(&value)
}

func (r *RegisteredContract5) AddIssuerFinancialInstitution() *BranchAndFinancialInstitutionIdentification5 {
	r.IssuerFinancialInstitution = new(BranchAndFinancialInstitutionIdentification5)
	return r.IssuerFinancialInstitution
}

func (r *RegisteredContract5) AddContract() *UnderlyingContract1Choice {
	r.Contract = new(UnderlyingContract1Choice)
	return r.Contract
}

func (r *RegisteredContract5) AddContractBalance() *ContractBalance1 {
	newValue := new(ContractBalance1)
	r.ContractBalance = append(r.ContractBalance, newValue)
	return newValue
}

func (r *RegisteredContract5) AddPaymentScheduleType() *PaymentScheduleType1Choice {
	r.PaymentScheduleType = new(PaymentScheduleType1Choice)
	return r.PaymentScheduleType
}

func (r *RegisteredContract5) AddRegisteredContractIdentification() *DocumentIdentification29 {
	r.RegisteredContractIdentification = new(DocumentIdentification29)
	return r.RegisteredContractIdentification
}

func (r *RegisteredContract5) AddPreviousRegisteredContractIdentification() *DocumentIdentification22 {
	r.PreviousRegisteredContractIdentification = new(DocumentIdentification22)
	return r.PreviousRegisteredContractIdentification
}

func (r *RegisteredContract5) AddRegisteredContractJournal() *RegisteredContractJournal1 {
	newValue := new(RegisteredContractJournal1)
	r.RegisteredContractJournal = append(r.RegisteredContractJournal, newValue)
	return newValue
}

func (r *RegisteredContract5) AddAmendment() *RegisteredContractAmendment1 {
	newValue := new(RegisteredContractAmendment1)
	r.Amendment = append(r.Amendment, newValue)
	return newValue
}

func (r *RegisteredContract5) AddSubmission() *RegisteredContractCommunication1 {
	r.Submission = new(RegisteredContractCommunication1)
	return r.Submission
}

func (r *RegisteredContract5) AddDelivery() *RegisteredContractCommunication1 {
	r.Delivery = new(RegisteredContractCommunication1)
	return r.Delivery
}

func (r *RegisteredContract5) SetLoanPrincipalAmount(value, currency string) {
	r.LoanPrincipalAmount = NewActiveCurrencyAndAmount(value, currency)
}

func (r *RegisteredContract5) SetEstimatedDateIndicator(value string) {
	r.EstimatedDateIndicator = (*TrueFalseIndicator)(&value)
}

func (r *RegisteredContract5) SetInterCompanyLoan(value string) {
	r.InterCompanyLoan = (*TrueFalseIndicator)(&value)
}

func (r *RegisteredContract5) SetAdditionalInformation(value string) {
	r.AdditionalInformation = (*Max1025Text)(&value)
}

redpajama_set_name: RedPajamaGithub


text:
package net.oneandone.troilus.api;

import java.nio.ByteBuffer;
import java.time.Duration;
import java.time.Instant;
import java.util.Iterator;
import java.util.Optional;

import net.oneandone.troilus.AbstractCassandraBasedTest;
import net.oneandone.troilus.Batch;
import net.oneandone.troilus.Batchable;
import net.oneandone.troilus.Count;
import net.oneandone.troilus.Dao;
import net.oneandone.troilus.DaoImpl;
import net.oneandone.troilus.IfConditionException;
import net.oneandone.troilus.Record;
import net.oneandone.troilus.CounterMutation;

import org.junit.Assert;
import org.junit.Test;

import com.datastax.driver.core.ConsistencyLevel;
import com.datastax.driver.core.ExecutionInfo;
import com.datastax.driver.core.querybuilder.QueryBuilder;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;

public class ColumnsApiTest extends AbstractCassandraBasedTest {

    @Test
    public void testSimpleTable() throws Exception {
        Dao usersDao = new DaoImpl(getSession(), UsersTable.TABLE)
                                 .withConsistency(ConsistencyLevel.LOCAL_QUORUM);

        Dao loginsDao = new DaoImpl(getSession(), LoginsTable.TABLE)
                                .withConsistency(ConsistencyLevel.LOCAL_QUORUM);

        Dao plusLoginsDao = new DaoImpl(getSession(), PlusLoginsTable.TABLE)
                                .withConsistency(ConsistencyLevel.LOCAL_QUORUM);

        Count num = usersDao.readAll()
                            .count()
                            .execute();
        Assert.assertEquals(0, num.getCount());

        ////////////////
        // inserts
        usersDao.writeWithKey(UsersTable.USER_ID, "95454")
                .value(UsersTable.IS_CUSTOMER, true)
                .value(UsersTable.PICTURE, ByteBuffer.wrap(new byte[] { 8, 4, 3}))
                .value(UsersTable.ADDRESSES, ImmutableList.of("stuttgart", "baden-baden"))
                .value(UsersTable.PHONE_NUMBERS, ImmutableSet.of("34234243", "9345324"))
                .execute();

        ExecutionInfo info =  usersDao.writeWithKey(UsersTable.USER_ID, "8345345")
                                      .value(UsersTable.PHONE_NUMBERS, ImmutableSet.of("24234244"))
                                      .value(UsersTable.IS_CUSTOMER, true)
                                      .ifNotExists()
                                      .withTtl(Duration.ofMinutes(2))
                                      .withWritetime(Instant.now().toEpochMilli() * 1000)
                                      .withTracking()
                                      .execute()
                                      .getExecutionInfo();

        Assert.assertNotNull(info.getQueryTrace());

        usersDao.writeWithKey(UsersTable.USER_ID, "4545")
                .value(UsersTable.IS_CUSTOMER, true)
                .value(UsersTable.PICTURE, ByteBuffer.wrap(new byte[] { 4, 5, 5}))
                .value(UsersTable.ADDRESSES, ImmutableList.of("münchen", "karlsruhe"))
                .value(UsersTable.PHONE_NUMBERS, ImmutableSet.of("94665", "34324543"))
                .ifNotExists()
                .withTracking()
                .execute();

        try {   // insert twice!
            usersDao.writeWithKey(UsersTable.USER_ID, "4545")
                    .value(UsersTable.IS_CUSTOMER, true)
                    .value(UsersTable.PICTURE, ByteBuffer.wrap(new byte[] { 4, 5, 5}))
                    .value(UsersTable.ADDRESSES, ImmutableList.of("münchen", "karlsruhe"))
                    .value(UsersTable.PHONE_NUMBERS, ImmutableSet.of("94665", "34324543"))
                    .ifNotExists()
                    .execute();

            Assert.fail("DuplicateEntryException expected");
        } catch (IfConditionException expected) { }

        usersDao.writeWithKey(UsersTable.USER_ID, "3434343")
                .value(UsersTable.IS_CUSTOMER, Optional.of(true))
                .value(UsersTable.PICTURE, ByteBuffer.wrap(new byte[] { 4, 5, 5}))
                .value(UsersTable.ADDRESSES, null)
                .value(UsersTable.PHONE_NUMBERS, Optional.empty())
                .execute();

        ////////////////
        // reads
        Optional<Record> optionalRecord = usersDao.readWithKey(UsersTable.USER_ID, "4545")
                                                  .column(UsersTable.PICTURE)
                                                  .column(UsersTable.ADDRESSES)
                                                  .column(UsersTable.PHONE_NUMBERS)
                                                  .execute();
        Assert.assertTrue(optionalRecord.isPresent());
        optionalRecord.ifPresent(record -> System.out.println(record.getList(UsersTable.ADDRESSES, String.class)));
        System.out.println(optionalRecord.get());

        Optional<Record> optionalRecord2 = usersDao.readWithKey(UsersTable.USER_ID, "95454")
                                                   .columns(UsersTable.PICTURE, UsersTable.ADDRESSES, UsersTable.PHONE_NUMBERS)
                                                   .execute();
        Assert.assertTrue(optionalRecord2.isPresent());
        optionalRecord2.ifPresent(record -> System.out.println(record.getList(UsersTable.ADDRESSES, String.class)));

        Optional<Record> optionalRecord3 = usersDao.readWithKey(UsersTable.USER_ID, "8345345")
                                                   .columnWithMetadata(UsersTable.IS_CUSTOMER)
                                                   .column(UsersTable.PICTURE)
                                                   .execute();
        Assert.assertTrue(optionalRecord3.isPresent());
        optionalRecord3.ifPresent(record -> System.out.println(record));

        num = usersDao.readAll()
                      .count()
                      .execute();
        Assert.assertEquals(4,  num.getCount());

        ////////////////
        // deletes
        usersDao.deleteWithKey(UsersTable.USER_ID, "4545")
                .execute();

        // check
        optionalRecord = usersDao.readWithKey(UsersTable.USER_ID, "4545")
                                 .column(UsersTable.USER_ID)
                                 .execute();
        Assert.assertFalse(optionalRecord.isPresent());

        ////////////////
        // batch inserts
        Batchable insert1 = usersDao.writeWithKey(UsersTable.USER_ID, "14323425")
                                    .value(UsersTable.IS_CUSTOMER, true)
                                    .value(UsersTable.ADDRESSES, ImmutableList.of("berlin", "budapest"))
                                    .value(UsersTable.PHONE_NUMBERS, ImmutableSet.of("12313241243", "232323"));

        Batchable insert2 = usersDao.writeWithKey(UsersTable.USER_ID, "2222")
                                    .value(UsersTable.IS_CUSTOMER, true)
                                    .value(UsersTable.ADDRESSES, ImmutableList.of("berlin", "budapest"))
                                    .value(UsersTable.PHONE_NUMBERS, ImmutableSet.of("12313241243", "232323"));

        usersDao.writeWithKey(UsersTable.USER_ID, "222222")
                .value(UsersTable.IS_CUSTOMER, true)
                .value(UsersTable.ADDRESSES, ImmutableList.of("hamburg"))
                .value(UsersTable.PHONE_NUMBERS, ImmutableSet.of("945453", "23432234"))
                .combinedWith(insert1)
                .combinedWith(insert2)
                .withWriteAheadLog()
                .execute();

        // check
        optionalRecord = usersDao.readWithKey(UsersTable.USER_ID, "14323425")
                                 .all()
                                 .execute();
        System.out.println(optionalRecord);
        Assert.assertTrue(optionalRecord.isPresent());

        Record record = usersDao.readWithKey(UsersTable.USER_ID, "8345345")
                                .execute()
                                .get();
        Assert.assertEquals("8345345", record.getString(UsersTable.USER_ID));
        Assert.assertEquals(true, record.getBool(UsersTable.IS_CUSTOMER));

        Iterator<String> phoneNumbers = record.getSet(UsersTable.PHONE_NUMBERS, String.class).iterator();
        Assert.assertEquals("24234244", phoneNumbers.next());
        Assert.assertFalse(phoneNumbers.hasNext());

        // Inc/Set counter value
        loginsDao.writeWithKey(LoginsTable.USER_ID, "8345345")
                .incr(LoginsTable.LOGINS)
                .execute();

        record = loginsDao.readWithKey(LoginsTable.USER_ID, "8345345")
                          .execute()
                          .get();
        Assert.assertEquals(1, record.getLong(LoginsTable.LOGINS));

        loginsDao.writeWithKey(LoginsTable.USER_ID, "8345345")
                 .incr(LoginsTable.LOGINS, 4)
                 .execute();

        record = loginsDao.readWithKey(LoginsTable.USER_ID, "8345345")
                          .execute()
                          .get();
        Assert.assertEquals(5, record.getLong(LoginsTable.LOGINS));

        loginsDao.writeWithKey(LoginsTable.USER_ID, "8345345")
                 .decr(LoginsTable.LOGINS)
                 .execute();

        record = loginsDao.readWithKey(LoginsTable.USER_ID, "8345345")
                          .execute()
                          .get();
        Assert.assertEquals(4, record.getLong(LoginsTable.LOGINS));

        loginsDao.writeWithKey(LoginsTable.USER_ID, "8345345")
                 .decr(LoginsTable.LOGINS)
                 .withConsistency(ConsistencyLevel.LOCAL_QUORUM)
                 .withWritetime(Instant.now().toEpochMilli() * 1000)
                 .execute();

        record = loginsDao.readWithKey(LoginsTable.USER_ID, "8345345")
                          .execute()
                          .get();
        Assert.assertEquals(3, record.getLong(LoginsTable.LOGINS));

        loginsDao.writeWhere(QueryBuilder.eq(LoginsTable.USER_ID, "8345345"))
                 .incr(LoginsTable.LOGINS)
                 .withWritetime(Instant.now().toEpochMilli() * 1000)
                 .withConsistency(ConsistencyLevel.LOCAL_QUORUM)
                 .execute();

        record = loginsDao.readWithKey(LoginsTable.USER_ID, "8345345")
                          .execute()
                          .get();
        Assert.assertEquals(4, record.getLong(LoginsTable.LOGINS));

        plusLoginsDao.writeWhere(QueryBuilder.eq(PlusLoginsTable.USER_ID, "8345345"))
                     .incr(PlusLoginsTable.LOGINS)
                     .withWritetime(Instant.now().toEpochMilli() * 1000)
                     .withConsistency(ConsistencyLevel.LOCAL_QUORUM)
                     .execute();

        record = plusLoginsDao.readWithKey(PlusLoginsTable.USER_ID, "8345345")
                          .execute()
                          .get();
        Assert.assertEquals(1, record.getLong(PlusLoginsTable.LOGINS));

        CounterMutation cm = loginsDao.writeWhere(QueryBuilder.eq(LoginsTable.USER_ID, "8345345"))
                                      .incr(LoginsTable.LOGINS);

        plusLoginsDao.writeWhere(QueryBuilder.eq(PlusLoginsTable.USER_ID, "8345345"))
                     .incr(PlusLoginsTable.LOGINS)
                     .withWritetime(Instant.now().toEpochMilli() * 1000)
                     .withConsistency(ConsistencyLevel.LOCAL_QUORUM)
                     .combinedWith(cm)
                     .execute();

        record = plusLoginsDao.readWithKey(PlusLoginsTable.USER_ID, "8345345")
                .execute()
                .get();
        Assert.assertEquals(2, record.getLong(PlusLoginsTable.LOGINS));

        record = loginsDao.readWithKey(LoginsTable.USER_ID, "8345345")
                .execute()
                .get();
        Assert.assertEquals(5, record.getLong(LoginsTable.LOGINS));

        // remove value
        usersDao.writeWithKey(UsersTable.USER_ID, "8345345")
                .value(UsersTable.IS_CUSTOMER, null)
                .execute();

        record = usersDao.readWithKey(UsersTable.USER_ID, "8345345")
                         .execute()
                         .get();
        Assert.assertEquals("8345345", record.getString(UsersTable.USER_ID));
        Assert.assertFalse(record.getBool(UsersTable.IS_CUSTOMER));

        phoneNumbers = record.getSet(UsersTable.PHONE_NUMBERS, String.class).iterator();
        Assert.assertEquals("24234244", phoneNumbers.next());
        Assert.assertFalse(phoneNumbers.hasNext());

        ////////////////////
        // conditional update

        try {
            usersDao.writeWithKey(UsersTable.USER_ID, "2222")
                    .value(UsersTable.ADDRESSES, ImmutableList.of("nürnberg"))
                    .onlyIf(QueryBuilder.eq(UsersTable.IS_CUSTOMER, false))
                    .withSerialConsistency(ConsistencyLevel.SERIAL)
                    .execute();
            Assert.fail("IfConditionException expected");
        } catch (IfConditionException expected) {  }

        record = usersDao.readWithKey(UsersTable.USER_ID, "2222")
                .execute()
                .get();

        Iterator<String> addresses= record.getList(UsersTable.ADDRESSES, String.class).iterator();
        Assert.assertEquals("berlin", addresses.next());
        Assert.assertEquals("budapest", addresses.next());
        Assert.assertFalse(addresses.hasNext());

        usersDao.writeWithKey(UsersTable.USER_ID, "2222")
                .value(UsersTable.ADDRESSES, ImmutableList.of("nürnberg"))
                .onlyIf(QueryBuilder.eq(UsersTable.IS_CUSTOMER, true))
                .execute();

        record = usersDao.readWithKey(UsersTable.USER_ID, "2222")
                .execute()
                .get();

        addresses= record.getList(UsersTable.ADDRESSES, String.class).iterator();
        Assert.assertEquals("nürnberg", addresses.next());
        Assert.assertFalse(addresses.hasNext());

        usersDao.writeWhere(QueryBuilder.in(UsersTable.USER_ID, "2222"))
                .value(UsersTable.ADDRESSES, ImmutableList.of("berlin", "budapest"))
                .execute();

        record = usersDao.readWithKey(UsersTable.USER_ID, "2222")
                .execute()
                .get();

        addresses= record.getList(UsersTable.ADDRESSES, String.class).iterator();
        Assert.assertEquals("berlin", addresses.next());
        Assert.assertEquals("budapest", addresses.next());
        Assert.assertFalse(addresses.hasNext());

        try {
            usersDao.deleteWithKey(UsersTable.USER_ID, "2222")
                    .onlyIf(QueryBuilder.eq(UsersTable.IS_CUSTOMER, false))
                    .execute();
            Assert.fail("IfConditionException expected");
        } catch (IfConditionException expected) { }

        Optional<Record> rec = usersDao.readWithKey(UsersTable.USER_ID, "2222")
                                       .execute();
        Assert.assertTrue(rec.isPresent());

        usersDao.deleteWithKey(UsersTable.USER_ID, "2222")
                .onlyIf(QueryBuilder.eq(UsersTable.IS_CUSTOMER, true))
                .execute();

        rec = usersDao.readWithKey(UsersTable.USER_ID, "2222")
                      .execute();
        Assert.assertFalse(rec.isPresent());

        usersDao.writeWithKey(UsersTable.USER_ID, "8345345")
                .value(UsersTable.IS_CUSTOMER, true)
                .value(UsersTable.ADDRESSES, ImmutableList.of("hamburg"))
                .execute();

        record = usersDao.readWithKey(UsersTable.USER_ID, "8345345")
                .execute()
                .get();
        Assert.assertEquals("8345345", record.getString(UsersTable.USER_ID));
        Assert.assertTrue(record.getBool(UsersTable.IS_CUSTOMER));
        Assert.assertFalse(record.getList(UsersTable.ADDRESSES, String.class).isEmpty());
        Assert.assertFalse(record.getSet(UsersTable.PHONE_NUMBERS, String.class).isEmpty());

        // remove value
        usersDao.writeWithKey(UsersTable.USER_ID, "8345345")
                .value(UsersTable.IS_CUSTOMER, null)
                .value(UsersTable.ADDRESSES, null)
                .execute();

        record = usersDao.readWithKey(UsersTable.USER_ID, "8345345")
                         .execute()
                         .get();
        Assert.assertEquals("8345345", record.getString(UsersTable.USER_ID));
        Assert.assertFalse(record.getBool(UsersTable.IS_CUSTOMER));
        Assert.assertTrue(record.getList(UsersTable.ADDRESSES, String.class).isEmpty());

        // add set value
        usersDao.writeWithKey(UsersTable.USER_ID, "8345345")
                .addSetValue(UsersTableFields.PHONE_NUMBERS, "12142343")
                .execute();

        record = usersDao.readWithKey(UsersTable.USER_ID, "8345345")
                         .execute()
                         .get();
        Assert.assertEquals("8345345", record.getString(UsersTable.USER_ID));
        Assert.assertFalse(record.getBool(UsersTable.IS_CUSTOMER));
        Assert.assertTrue(record.getSet(UsersTable.PHONE_NUMBERS, String.class).contains("12142343"));

        // add set value
        usersDao.writeWithKey(UsersTable.USER_ID, "8345345")
                .addSetValue(UsersTable.PHONE_NUMBERS, "23234234")
                .execute();

        record = usersDao.readWithKey(UsersTable.USER_ID, "8345345")
                         .execute()
                         .get();
        Assert.assertEquals("8345345", record.getString(UsersTable.USER_ID));
        Assert.assertFalse(record.getBool(UsersTable.IS_CUSTOMER));
        Assert.assertTrue(record.getSet(UsersTable.PHONE_NUMBERS, String.class).contains("12142343"));
        Assert.assertTrue(record.getSet(UsersTable.PHONE_NUMBERS, String.class).contains("23234234"));

        // remove set value
        usersDao.writeWithKey(UsersTable.USER_ID, "8345345")
                .removeSetValue(UsersTable.PHONE_NUMBERS, "12142343")
                .execute();

        record = usersDao.readWithKey(UsersTable.USER_ID, "8345345")
                         .execute()
                         .get();
        Assert.assertEquals("8345345", record.getString(UsersTable.USER_ID));
        Assert.assertFalse(record.getBool(UsersTable.IS_CUSTOMER));
        Assert.assertFalse(record.getSet(UsersTable.PHONE_NUMBERS, String.class).contains("12142343"));
        Assert.assertTrue(record.getSet(UsersTable.PHONE_NUMBERS, String.class).contains("23234234"));
        Assert.assertTrue(record.getList(UsersTable.ADDRESSES, String.class).isEmpty());

        // add list value
        usersDao.writeWithKey(UsersTable.USER_ID, "8345345")
                .value(UsersTable.ADDRESSES, ImmutableList.of("bonn", "stuttgart"))
                .execute();

        record = usersDao.readWithKey(UsersTable.USER_ID, "8345345")
                         .execute()
                         .get();
        Assert.assertEquals("8345345", record.getString(UsersTable.USER_ID));
        Assert.assertFalse(record.getBool(UsersTable.IS_CUSTOMER));
        Iterator<String> addrIt = record.getList(UsersTable.ADDRESSES, String.class).iterator();
        Assert.assertEquals("bonn", addrIt.next());
        Assert.assertEquals("stuttgart", addrIt.next());
        Assert.assertFalse(addrIt.hasNext());

        // modify list value
        usersDao.writeWithKey(UsersTable.USER_ID, "8345345")
                .prependListValue(UsersTable.ADDRESSES, "bern")
                .prependListValue(UsersTable.ADDRESSES, "neustadt")
                .prependListValue(UsersTable.ADDRESSES, "ulm")
                .appendListValue(UsersTable.ADDRESSES, "frankfurt")
                .appendListValue(UsersTable.ADDRESSES, "innsbruck")
                .removeListValue(UsersTable.ADDRESSES, "bonn")
                .appendListValue(UsersTable.ADDRESSES, "berlin")
                .execute();

        record = usersDao.readWithKey(UsersTable.USER_ID, "8345345")
                         .execute()
                         .get();
        Assert.assertEquals("8345345", record.getString(UsersTable.USER_ID));
        Assert.assertFalse(record.getBool(UsersTable.IS_CUSTOMER));
        addrIt = record.getList(UsersTable.ADDRESSES, String.class).iterator();
        Assert.assertEquals("ulm", addrIt.next());
        Assert.assertEquals("neustadt", addrIt.next());
        Assert.assertEquals("bern", addrIt.next());
        Assert.assertEquals("stuttgart", addrIt.next());
        Assert.assertEquals("frankfurt", addrIt.next());
        Assert.assertEquals("innsbruck", addrIt.next());
        Assert.assertEquals("berlin", addrIt.next());
        Assert.assertFalse(addrIt.hasNext());

        // set map
        usersDao.writeWithKey(UsersTable.USER_ID, "8345345")
                .value(UsersTable.ROLES, ImmutableMap.of("customer", "xe333"))
                .execute();

        record = usersDao.readWithKey(UsersTable.USER_ID, "8345345")
                         .execute()
                         .get();
        Assert.assertEquals("8345345", record.getString(UsersTable.USER_ID));
        Assert.assertEquals("xe333", record.getMap(UsersTable.ROLES, String.class, String.class).get("customer"));

        usersDao.writeWithKey(UsersTable.USER_ID, "8345345")
                .putMapValue(UsersTable.ROLES, "player_type1", "p14334")
                .putMapValue(UsersTable.ROLES, "player_type2", "p233")
                .execute();

        record = usersDao.readWithKey(UsersTable.USER_ID, "8345345")
                         .execute()
                         .get();
        Assert.assertEquals("xe333", record.getMap(UsersTable.ROLES, String.class, String.class).get("customer"));
        Assert.assertEquals("p14334", record.getMap(UsersTable.ROLES, String.class, String.class).get("player_type1"));
        Assert.assertEquals("p233", record.getMap(UsersTable.ROLES, String.class, String.class).get("player_type2"));

   /* does not work
        usersDao.writeWithKey(UsersTable.USER_ID, "8345345")
                .putMapValue(UsersTable.ROLES, "player_type1", null)
                .execute();

        record = usersDao.readWithKey(UsersTable.USER_ID, "8345345")
                         .execute()
                         .get();
                         */

        usersDao.writeWithKey(UsersTableFields.USER_ID, "343434")
                .value(UsersTableFields.IS_CUSTOMER, true)
                .value(UsersTable.ROLES, ImmutableMap.of("customer", "xe333"))
                .value(UsersTableFields.PICTURE, new byte[] { 4, 5, 5})
                .value(UsersTableFields.ADDRESSES, ImmutableList.of("münchen", "karlsruhe"))
                .value(UsersTableFields.PHONE_NUMBERS, ImmutableSet.of("94665", "34324543"))
                .ifNotExists()
                .withTracking()
                .execute();

        record = usersDao.readWithKey(UsersTableFields.USER_ID, "343434")
                .execute()
                .get();

        Assert.assertEquals("343434", record.getValue(UsersTableFields.USER_ID));
        Assert.assertNotNull(record.getValue(UsersTableFields.IS_CUSTOMER));
        Assert.assertFalse(record.getValue(UsersTableFields.PHONE_NUMBERS).contains("12142343"));
        Assert.assertTrue(record.getValue(UsersTableFields.PHONE_NUMBERS).contains("34324543"));
        Assert.assertTrue(record.getValue(UsersTableFields.ADDRESSES).contains("karlsruhe"));
        Assert.assertEquals("xe333", record.getValue(UsersTableFields.ROLES).get("customer"));
        Assert.assertArrayEquals(new byte[] { 4, 5, 5}, record.getValue(UsersTableFields.PICTURE));

        usersDao.writeWithKey(UsersTableFields.USER_ID, "45436")
                .value(UsersTableFields.PHONE_NUMBERS, ImmutableSet.of("24234244"))
                .value(UsersTableFields.IS_CUSTOMER, true)
                .value(UsersTableFields.USER_TYPE, UserType.METAL)
                .execute();

        Assert.assertNotNull(info.getQueryTrace());

        record = usersDao.readWithKey(UsersTableFields.USER_ID, "45436")
                .execute()
                .get();

        Assert.assertEquals(UserType.METAL, record.getValue(UsersTableFields.USER_TYPE));

        record = usersDao.withTracking().readWithKey(UsersTableFields.USER_ID, "45436")
                         .withConsistency(ConsistencyLevel.ALL)
                         .execute()
                         .get();

        Assert.assertEquals(UserType.METAL, record.getValue(UsersTableFields.USER_TYPE));
        Assert.assertNotNull(record.getExecutionInfo().getQueryTrace());
  //      Assert.assertTrue(record.toString().contains("Merging memtable tombstones"));

        record = usersDao.withoutTracking()
                         .readWithKey(UsersTableFields.USER_ID, "45436")
                         .execute()
                         .get();

        Assert.assertEquals(UserType.METAL, record.getValue(UsersTableFields.USER_TYPE));
        Assert.assertNull(record.getExecutionInfo().getQueryTrace());

        usersDao.writeWithKey(UsersTable.USER_ID, "23452342342")
                .value(UsersTable.IS_CUSTOMER, true)
                .execute();

        record = usersDao.readWithKey(UsersTableFields.USER_ID, "23452342342")
                .execute()
                .get();

        Assert.assertTrue(record.getValue(UsersTableFields.PHONE_NUMBERS).isEmpty());
        Assert.assertTrue(record.getValue(UsersTableFields.ADDRESSES).isEmpty());
        Assert.assertArrayEquals(new byte[0], record.getValue(UsersTableFields.PICTURE));
        Assert.assertNull(record.getValue(UsersTableFields.USER_TYPE));

        record = usersDao.readWithKey(UsersTableFields.USER_ID, "23452342342")
                         .column(UsersTable.USER_TYPE)
                         .execute()
                         .get();

        try {
            Assert.assertNull(record.getBool(UsersTableFields.IS_CUSTOMER.getName()));
            Assert.fail("IllegalArgumentException expected");
        } catch (IllegalArgumentException expected) { }

        usersDao.writeWithKey(UsersTable.USER_ID, "5553344")
                .value(UsersTable.IS_CUSTOMER, true)
                .withTtl(Duration.ofSeconds(1))
                .execute();

        Batchable insert5 = usersDao.writeWithKey(UsersTable.USER_ID, "234234234424")
                                    .value(UsersTable.IS_CUSTOMER, true)
                                    .value(UsersTable.ADDRESSES, ImmutableList.of("berlin", "budapest"))
                                    .value(UsersTable.PHONE_NUMBERS, ImmutableSet.of("12313241243", "232323"));

        usersDao.writeWithKey(UsersTable.USER_ID, "235423423424")
                .value(UsersTable.IS_CUSTOMER, true)
                .value(UsersTable.ADDRESSES, ImmutableList.of("hamburg"))
                .value(UsersTable.PHONE_NUMBERS, ImmutableSet.of("945453", "23432234"))
                .combinedWith(insert5)
                .withoutWriteAheadLog()
                .withConsistency(ConsistencyLevel.QUORUM)
                .execute();

        Batchable w1 = usersDao.writeWithKey(UsersTable.USER_ID, "456456645243245")
                               .value(UsersTable.IS_CUSTOMER, true)
                               .value(UsersTable.ADDRESSES, ImmutableList.of("berlin", "budapest"))
                               .value(UsersTable.PHONE_NUMBERS, ImmutableSet.of("12313241243", "232323"));

        Batchable w2 = usersDao.writeWithKey(UsersTable.USER_ID, "456456645243245")
                               .value(UsersTable.IS_CUSTOMER, true)
                               .value(UsersTable.ADDRESSES, ImmutableList.of("berlin", "budapest"))
                               .value(UsersTable.PHONE_NUMBERS, ImmutableSet.of("12313241243", "232323"));

        Batch batch = w1.combinedWith(w2);
        batch.execute();

     //   Batchable batchable = w1.combinedWith(w2);
     //   batchable.execute();

    }
}


text:
package com.crowdin.cli.commands.actions;

import com.crowdin.cli.client.ClientGlossary;
import com.crowdin.cli.commands.NewAction;
import com.crowdin.cli.commands.Outputter;
import com.crowdin.cli.properties.BaseProperties;
import com.crowdin.cli.properties.NewBasePropertiesUtilBuilder;
import com.crowdin.client.glossaries.model.AddGlossaryRequest;
import com.crowdin.client.glossaries.model.Glossary;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

import java.io.File;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;

import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;

public class GlossaryUploadActionTest {

    private static final Long glossaryId = 42L;
    private static final Long glossaryIdNotExist = 41L;
    private static final Long storageId = 52L;
    private static final String glossaryNameUnique = "uniqueGlossary";
    private static final String glossaryNameRepeated = "twoGlossariesWithSameName";
    private static final String glossaryNameNotExist = "notExistentGlossary";
    BaseProperties pb = NewBasePropertiesUtilBuilder
        .minimalBuilt()
        .build();
    File file = new File(getClass().getClassLoader().getResource("file.txt").getFile());
    File fileNotExist = new File("nowhere.txt");

    private ClientGlossary clientMock;
    private NewAction<BaseProperties, ClientGlossary> action;

    @BeforeEach
    public void beforeEach() {
        clientMock = mock(ClientGlossary.class);

        List<Glossary> glossaries = Arrays.asList(
            new Glossary() {{
                    setId(glossaryId);
                    setName(glossaryNameUnique);
                }},
            new Glossary() {{
                    setId(glossaryId + 1);
                    setName(glossaryNameRepeated);
                }},
            new Glossary() {{
                    setId(glossaryId + 2);
                    setName(glossaryNameRepeated);
                }}
        );
        Glossary glossary = new Glossary() {{
                setId(glossaryId);
                setName("New glossary");
            }};
        when(clientMock.getGlossary(eq(glossaryIdNotExist)))
            .thenReturn(Optional.empty());
        when(clientMock.getGlossary(eq(glossaryId)))
            .thenReturn(Optional.of(glossary));
        when(clientMock.listGlossaries())
            .thenReturn(glossaries);
        when(clientMock.uploadStorage(any(), any()))
            .thenReturn(storageId);
        when(clientMock.addGlossary(any()))
            .thenAnswer((invocationOnMock) -> {
                glossary.setName(((AddGlossaryRequest) invocationOnMock.getArgument(0)).getName());
                return glossary;
            });
    }

    @Test
    public void test_withId() {
        action = new GlossaryUploadAction(file, glossaryId, null, null, null, null);
        action.act(Outputter.getDefault(), pb, clientMock);

        verify(clientMock).getGlossary(eq(glossaryId));
        verify(clientMock).uploadStorage(any(), any());
        verify(clientMock).importGlossary(eq(glossaryId), any());
        verifyNoMoreInteractions(clientMock);
    }

    @Test
    public void test_withName() {
        action = new GlossaryUploadAction(file, null, glossaryNameUnique, null, null, null);
        action.act(Outputter.getDefault(), pb, clientMock);

        verify(clientMock).listGlossaries();
        verify(clientMock).uploadStorage(any(), any());
        verify(clientMock).importGlossary(eq(glossaryId), any());
        verifyNoMoreInteractions(clientMock);
    }

    @Test
    public void test_noIdentifiers_createNew() {
        action = new GlossaryUploadAction(file, null, null, null, null, null);
        action.act(Outputter.getDefault(), pb, clientMock);

        verify(clientMock).addGlossary(any());
        verify(clientMock).uploadStorage(any(), any());
        verify(clientMock).importGlossary(eq(glossaryId), any());
        verifyNoMoreInteractions(clientMock);
    }

    @Test
    public void test_withName_notFound_createNew() {
        action = new GlossaryUploadAction(file, null, glossaryNameNotExist, null, null, null);
        assertThrows(RuntimeException.class, () -> action.act(Outputter.getDefault(), pb, clientMock));

        verify(clientMock).listGlossaries();
        verifyNoMoreInteractions(clientMock);
    }

    @Test
    public void test_withId_throwsNotFound() {
        action = new GlossaryUploadAction(file, glossaryIdNotExist, null, null, null, null);
        assertThrows(RuntimeException.class, () -> action.act(Outputter.getDefault(), pb, clientMock));

        verify(clientMock).getGlossary(eq(glossaryIdNotExist));
        verifyNoMoreInteractions(clientMock);
    }

    @Test
    public void test_withName_throwsTooManyWithTargetName() {
        action = new GlossaryUploadAction(file, null, glossaryNameRepeated, null, null, null);
        assertThrows(RuntimeException.class, () -> action.act(Outputter.getDefault(), pb, clientMock));

        verify(clientMock).listGlossaries();
        verifyNoMoreInteractions(clientMock);
    }

    @Test
    public void test_standard_throwsFileNotFound() {
        action = new GlossaryUploadAction(fileNotExist, glossaryId, null, null, null, null);
        assertThrows(RuntimeException.class, () -> action.act(Outputter.getDefault(), pb, clientMock));

        verify(clientMock).getGlossary(eq(glossaryId));
        verifyNoMoreInteractions(clientMock);
    }
}


text:
"""MSG Builder."""

from typing import Iterator, List, Optional
import acme
from acme import adders
from acme import core
from acme import specs
from acme.agents.jax import actors
from acme.agents.jax import builders
from acme.jax import networks as networks_lib
from acme.jax import variable_utils
from acme.utils import counting
from acme.utils import loggers
import reverb
from jrl.agents.msg import config as msg_config
from jrl.agents.msg import learning
from jrl.agents.msg import networks as msg_networks

class MSGBuilder(builders.ActorLearnerBuilder):
  """MSG Builder."""

  def __init__(
      self,
      config,
      # make_demonstrations: Callable[[int], Iterator[types.Transition]],
      make_demonstrations,
      ):
    self._config = config
    self._make_demonstrations = make_demonstrations

  def make_learner(
      self,
      random_key,
      networks,
      dataset,
      logger_fn,
      replay_client = None,
      counter = None,
      checkpoint = False,
  ):
    del dataset  # Offline RL

    data_iter = self._make_demonstrations()

    return learning.MSGLearner(
        ensemble_size=self._config.ensemble_size,
        beta=self._config.beta,
        networks=networks,
        rng=random_key,
        iterator=data_iter,
        ensemble_method=self._config.ensemble_method,
        perform_sarsa_q_eval=self._config.perform_sarsa_q_eval,
        num_q_repr_pretrain_iters=self._config.num_q_repr_pretrain_iters,
        pretrain_temp=self._config.pretrain_temp,
        use_sass=self._config.use_sass,
        num_bc_iters=self._config.num_bc_iters,
        use_random_weighting_in_critic_loss=(
            self._config.use_random_weighting_in_critic_loss),
        use_ema_target_critic_params=self._config.use_ema_target_critic_params,
        entropy_coefficient=self._config.entropy_coefficient,
        target_entropy=self._config.target_entropy,
        use_entropy_regularization=self._config.use_entropy_regularization,
        behavior_regularization_type=self._config.behavior_regularization_type,
        behavior_regularization_alpha=(
            self._config.behavior_regularization_alpha),
        num_cql_actions=self._config.num_cql_actions,
        td_target_method=self._config.td_target_method,
        critic_random_init=self._config.critic_random_init,
        use_img_encoder=self._config.use_img_encoder,
        img_encoder_params_ckpt_path=self._config.img_encoder_params_ckpt_path,
        rem_mode=self._config.rem_mode,
        mimo_using_adamw=self._config.mimo_using_adamw,
        mimo_using_obs_tile=self._config.mimo_using_obs_tile,
        mimo_using_act_tile=self._config.mimo_using_act_tile,
        policy_lr=self._config.policy_lr,
        q_lr=self._config.q_lr,
        counter=counter,
        logger=logger_fn('learner'),
        num_sgd_steps_per_step=self._config.num_sgd_steps_per_step,)

  def make_actor(
      self,
      random_key,
      policy_network,
      adder = None,
      variable_source = None,
      force_eval_with_q_filter=False):
    assert variable_source is not None
    if self._config.eval_with_q_filter or force_eval_with_q_filter:
      params_to_get = ['policy', 'all_q']
      if self._config.use_img_encoder:
        params_to_get.append('img_encoder')
      return actors.GenericActor(
          actor=policy_network,
          random_key=random_key,
          # Inference happens on CPU, so it's better to move variables there.
          variable_client=variable_utils.VariableClient(
              variable_source, params_to_get, device='cpu'),
          adder=adder,
      )
    else:
      params_to_get = ['policy']
      if self._config.use_img_encoder:
        params_to_get.append('img_encoder')
      return actors.GenericActor(
          actor=policy_network,
          random_key=random_key,
          # Inference happens on CPU, so it's better to move variables there.
          variable_client=variable_utils.VariableClient(
              variable_source, params_to_get, device='cpu'),
          adder=adder,
      )

  def make_replay_tables(
      self,
      environment_spec,
      policy
  ):
    """Create tables to insert data into."""
    return []

  def make_dataset_iterator(
      self,
      replay_client):
    """Create a dataset iterator to use for learning/updating the agent."""
    return None

  def make_adder(self,
                 replay_client):
    """Create an adder which records data generated by the actor/environment."""
    return None


text:
<!DOCTYPE html>

    <title>Gadget Catalog</title>

    <meta charset="utf-8">
    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
    <meta http-equiv="x-ua-compatible" content="ie=edge">
    <meta name="description" content="A react application for cataloging gadgets.">
    <link rel="shortcut icon" href="images/favicon.png">

    <meta property="og:url" content="https://gadget-catalog-io.herokuapp.com/"/>
    <meta property="og:title" content="Gadget Catalog"/>
    <meta property="og:description" content="A single page application for cataloging gadgets. Built with react, express, mongodb and much more."/>
    <meta property="og:image" content="https://res.cloudinary.com/shibbir/image/upload/v1572199793/gadget-catalog/blue-desktop-with-laptop-compact-discs.jpg"/>

    {{#each cssFiles}}<link rel="stylesheet" href="/{{this}}">{{/each}}

                Loading...

    {{#each jsFiles}}{{/each}}


text:
using Microsoft.AspNetCore.Mvc;
using Stubby.AzureBlobStorage.Engine;
using System;
using System.IO;
using System.Linq;
using System.Security.Cryptography;

namespace Stubby.AzureBlobStorage.Controllers
{
    public class BlobController : ControllerBase
    {
        private readonly IBlobServiceEngine _engine;

        public BlobController(IBlobServiceEngine engine)
        {
            _engine = engine;
        }

        [HttpHead]
        public IActionResult Head(String container, String blob)
        {
            if (_engine.ContainsBlob(container, blob))
            {
                var bytes = _engine.GetBlob(container, blob);
                HttpContext.Response.Headers.ContentLength = bytes.Length;

                HttpContext.Response.ContentType = "application/octet-stream";
                HttpContext.Response.Headers.Add("Content-MD5", GetMd5Hash(bytes));
                HttpContext.Response.Headers.Add("Accept-Ranges", "bytes");
                HttpContext.Response.Headers.Add("Last-Modified", "Fri, 19 Aug 2016 11:37:26 GMT");
                HttpContext.Response.Headers.Add("ETag", "0x8D3C825319A7E5B");
                HttpContext.Response.Headers.Add("x-ms-request-id", "57649b7b-001e-003f-45b1-20a574000000");
                if ("2017-04-17" == HttpContext.Request.Headers["x-ms-version"].First())
                {
                    HttpContext.Response.Headers.Add("x-ms-version", "2017-04-17");
                }
                else
                {
                    HttpContext.Response.Headers.Add("x-ms-version:", "2016-05-31");
                }
                HttpContext.Response.Headers.Add("x-ms-lease-status", "unlocked");
                HttpContext.Response.Headers.Add("x-ms-lease-state", "available");
                HttpContext.Response.Headers.Add("x-ms-blob-type", "BlockBlob");
                HttpContext.Response.Headers.Add("x-ms-server-encrypted", "false");
                HttpContext.Response.Headers.Add("Date", "Tue, 29 Aug 2017 10:30:04 GMT");
                return new StatusCodeResult(200);
            }
            else
            {
                return new NotFoundResult();
            }
        }

        [HttpGet]
        public IActionResult GetBlokList(String container, String blob, String blocklisttype)
        {
            if (_engine.ContainsBlob(container, blob))
            {
                HttpContext.Response.StatusCode = 200;
                HttpContext.Response.Headers.Add("Content-Type", "application/xml");
                HttpContext.Response.Headers.Add("x-ms-request-id", Guid.NewGuid().ToString("D"));
                HttpContext.Response.Headers.Add("Date", DateTime.UtcNow.ToString("R"));

                var tagName = "UncommittedBlocks";
                var content = @"<?xml version=""1.0"" encoding=""utf-8""?>
<BlockList>
    <" + tagName + @"/>
</BlockList>";

                HttpContext.Response.Headers.ContentLength = content.Length;
                return Content(content);
            }
            else
            {
                return new NotFoundResult();
            }
        }

        [HttpGet]
        public IActionResult Get(String container, String blob)
        {
            if (_engine.ContainsBlob(container, blob))
            {
                var bytes = _engine.GetBlob(container, blob);
                HttpContext.Response.Headers.ContentLength = bytes.Length;

                if ("2017-04-17" == HttpContext.Request.Headers["x-ms-version"].First())
                {
                    if (HttpContext.Request.Headers.ContainsKey("x-ms-range"))
                    {
                        HttpContext.Response.StatusCode = 206;
                    }
                    else
                    {
                        HttpContext.Response.StatusCode = 200;
                    }
                    HttpContext.Response.Headers.Add("Content-Range", $"bytes 0-{bytes.Length - 1}/{bytes.Length}");
                    HttpContext.Response.Headers.Add("x-ms-blob-content-md5", GetMd5Hash(bytes));
                    HttpContext.Response.Headers.Add("Accept-Ranges", "bytes");
                    HttpContext.Response.Headers.Add("Last-Modified", DateTime.UtcNow.ToString("R"));

                    HttpContext.Response.Headers.Add("ETag", "0x8D3C825319A7E5B");
                    HttpContext.Response.Headers.Add("x-ms-request-id", Guid.NewGuid().ToString("D"));
                    HttpContext.Response.Headers.Add("x-ms-version", "2017-04-17");
                    HttpContext.Response.Headers.Add("x-ms-lease-status", "unlocked");
                    HttpContext.Response.Headers.Add("x-ms-lease-state", "available");
                    HttpContext.Response.Headers.Add("x-ms-blob-type", "BlockBlob");
                    HttpContext.Response.Headers.Add("x-ms-server-encrypted", "false");
                    HttpContext.Response.Headers.Add("Date", DateTime.UtcNow.ToString("R"));
                }
                else
                {
                    HttpContext.Response.StatusCode = 200;
                    HttpContext.Response.Headers.Add("Content-MD5", GetMd5Hash(bytes));
                    HttpContext.Response.Headers.Add("Accept-Ranges", "bytes");
                    HttpContext.Response.Headers.Add("Last-Modified", DateTime.UtcNow.ToString("R"));

                    HttpContext.Response.Headers.Add("ETag", "0x8D3C825319A7E5B");
                    HttpContext.Response.Headers.Add("x-ms-request-id:", Guid.NewGuid().ToString("D"));
                    HttpContext.Response.Headers.Add("x-ms-version:", "2016-05-31");
                    HttpContext.Response.Headers.Add("x-ms-lease-status", "unlocked");
                    HttpContext.Response.Headers.Add("x-ms-lease-state", "available");
                    HttpContext.Response.Headers.Add("x-ms-blob-type", "BlockBlob");
                    HttpContext.Response.Headers.Add("x-ms-server-encrypted", "false");
                    HttpContext.Response.Headers.Add("Date", DateTime.UtcNow.ToString("R"));
                }
                return File(bytes, "application/octet-stream");
            }
            else
            {
                return new NotFoundResult();
            }
        }

        [HttpPut]
        public IActionResult Put(String container, String blob)
        {
            byte[] content;
            using (MemoryStream stream = new MemoryStream())
            {
                Request.Body.CopyTo(stream);
                content = stream.ToArray();
            }

            _engine.PutBlob(container, blob, content);

            HttpContext.Response.Headers.Add("Content-MD5", GetMd5Hash(content));
            HttpContext.Response.Headers.Add("Last-Modified", DateTime.UtcNow.ToString("R"));
            HttpContext.Response.Headers.Add("x-ms-request-id", Guid.NewGuid().ToString("D"));
            HttpContext.Response.Headers.Add("Date", DateTime.UtcNow.ToString("R"));
            HttpContext.Response.Headers.Add("ETag", "0x8D3C825319A7E5B");
            HttpContext.Response.Headers.Add("x-ms-version", HttpContext.Request.Headers["x-ms-version"].First());
            HttpContext.Response.Headers.Add("x-ms-server-encrypted", "false");

            return new StatusCodeResult(201);
        }

        public static string GetMd5Hash(byte[] input)
        {
            using (var md5Check = MD5.Create())
            {
                md5Check.TransformBlock(input, 0, input.Length, null, 0);
                md5Check.TransformFinalBlock(new byte[0], 0, 0);

                byte[] hashBytes = md5Check.Hash;
                return Convert.ToBase64String(hashBytes);
            }
        }
    }
}


text:
<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.exmo</groupId>
    <artifactId>ws-client-example</artifactId>
    <version>1.0-SNAPSHOT</version>

    <name>ws-client-example</name>
    <url>https://github.com/exmo-dev/exmo_api_lib</url>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <maven.compiler.source>1.8</maven.compiler.source>
        <maven.compiler.target>1.8</maven.compiler.target>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.java-websocket</groupId>
            <artifactId>Java-WebSocket</artifactId>
            <version>1.5.1</version>
        </dependency>

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.11</version>
            <scope>test</scope>
        </dependency>
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <artifactId>maven-assembly-plugin</artifactId>
                <configuration>
                    <archive>
                        <manifest>
                            <mainClass>com.exmo.App</mainClass>
                        </manifest>
                    </archive>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
            </plugin>
        </plugins>
    </build>
</project>


text:
var __extends = (this && this.__extends) || function (d, b) {
    for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
    function __() { this.constructor = d; }
    d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
import { Injectable } from '@angular/core';
import { isPresent } from '../src/facade/lang';
import { RequestMethod } from './enums';
import { Headers } from './headers';
import { normalizeMethodName } from './http_utils';
import { URLSearchParams } from './url_search_params';
/**
 * Creates a request options object to be optionally provided when instantiating a
 * {@link Request}.
 *
 * This class is based on the `RequestInit` description in the [Fetch
 * Spec](https://fetch.spec.whatwg.org/#requestinit).
 *
 * All values are null by default. Typical defaults can be found in the {@link BaseRequestOptions}
 * class, which sub-classes `RequestOptions`.
 *
 * ### Example ([live demo](http://plnkr.co/edit/7Wvi3lfLq41aQPKlxB4O?p=preview))
 *
 * ```typescript
 * import {RequestOptions, Request, RequestMethod} from '@angular/http';
 *
 * var options = new RequestOptions({
 *   method: RequestMethod.Post,
 *   url: 'https://google.com'
 * });
 * var req = new Request(options);
 * console.log('req.method:', RequestMethod[req.method]); // Post
 * console.log('options.url:', options.url); // https://google.com
 * ```
 *
 * @experimental
 */
export var RequestOptions = (function () {
    function RequestOptions(_a) {
        var _b = _a === void 0 ? {} : _a, method = _b.method, headers = _b.headers, body = _b.body, url = _b.url, search = _b.search, withCredentials = _b.withCredentials, responseType = _b.responseType;
        this.method = isPresent(method) ? normalizeMethodName(method) : null;
        this.headers = isPresent(headers) ? headers : null;
        this.body = isPresent(body) ? body : null;
        this.url = isPresent(url) ? url : null;
        this.search = isPresent(search) ?
            (typeof search === 'string' ? new URLSearchParams((search)) :
                (search)) :
            null;
        this.withCredentials = isPresent(withCredentials) ? withCredentials : null;
        this.responseType = isPresent(responseType) ? responseType : null;
    }
    /**
     * Creates a copy of the `RequestOptions` instance, using the optional input as values to override
     * existing values. This method will not change the values of the instance on which it is being
     * called.
     *
     * Note that `headers` and `search` will override existing values completely if present in
     * the `options` object. If these values should be merged, it should be done prior to calling
     * `merge` on the `RequestOptions` instance.
     *
     * ### Example ([live demo](http://plnkr.co/edit/6w8XA8YTkDRcPYpdB9dk?p=preview))
     *
     * ```typescript
     * import {RequestOptions, Request, RequestMethod} from '@angular/http';
     *
     * var options = new RequestOptions({
     *   method: RequestMethod.Post
     * });
     * var req = new Request(options.merge({
     *   url: 'https://google.com'
     * }));
     * console.log('req.method:', RequestMethod[req.method]); // Post
     * console.log('options.url:', options.url); // null
     * console.log('req.url:', req.url); // https://google.com
     * ```
     */
    RequestOptions.prototype.merge = function (options) {
        return new RequestOptions({
            method: options && isPresent(options.method) ? options.method : this.method,
            headers: options && isPresent(options.headers) ? options.headers : this.headers,
            body: options && isPresent(options.body) ? options.body : this.body,
            url: options && isPresent(options.url) ? options.url : this.url,
            search: options && isPresent(options.search) ?
                (typeof options.search === 'string' ? new URLSearchParams(options.search) :
                    (options.search).clone()) :
                this.search,
            withCredentials: options && isPresent(options.withCredentials) ? options.withCredentials :
                this.withCredentials,
            responseType: options && isPresent(options.responseType) ? options.responseType :
                this.responseType
        });
    };
    return RequestOptions;
}());
/**
 * Subclass of {@link RequestOptions}, with default values.
 *
 * Default values:
 *  * method: {@link RequestMethod RequestMethod.Get}
 *  * headers: empty {@link Headers} object
 *
 * This class could be extended and bound to the {@link RequestOptions} class
 * when configuring an {@link Injector}, in order to override the default options
 * used by {@link Http} to create and send {@link Request Requests}.
 *
 * ### Example ([live demo](http://plnkr.co/edit/LEKVSx?p=preview))
 *
 * ```typescript
 * import {provide} from '@angular/core';
 * import {bootstrap} from '@angular/platform-browser/browser';
 * import {HTTP_PROVIDERS, Http, BaseRequestOptions, RequestOptions} from '@angular/http';
 * import {App} from './myapp';
 *
 * class MyOptions extends BaseRequestOptions {
 *   search: string = 'coreTeam=true';
 * }
 *
 * bootstrap(App, [HTTP_PROVIDERS, {provide: RequestOptions, useClass: MyOptions}]);
 * ```
 *
 * The options could also be extended when manually creating a {@link Request}
 * object.
 *
 * ### Example ([live demo](http://plnkr.co/edit/oyBoEvNtDhOSfi9YxaVb?p=preview))
 *
 * ```
 * import {BaseRequestOptions, Request, RequestMethod} from '@angular/http';
 *
 * var options = new BaseRequestOptions();
 * var req = new Request(options.merge({
 *   method: RequestMethod.Post,
 *   url: 'https://google.com'
 * }));
 * console.log('req.method:', RequestMethod[req.method]); // Post
 * console.log('options.url:', options.url); // null
 * console.log('req.url:', req.url); // https://google.com
 * ```
 *
 * @experimental
 */
export var BaseRequestOptions = (function (_super) {
    __extends(BaseRequestOptions, _super);
    function BaseRequestOptions() {
        _super.call(this, { method: RequestMethod.Get, headers: new Headers() });
    }
    BaseRequestOptions.decorators = [
        { type: Injectable },
    ];
    /** @nocollapse */
    BaseRequestOptions.ctorParameters = [];
    return BaseRequestOptions;
}(RequestOptions));
//# sourceMappingURL=base_request_options.js.map


text:
<!DOCTYPE html>

    <meta charset="UTF-8" />
    <meta name="robots" content="index, follow, all" />
    <title>Thelia\Controller\Admin\AbstractSeoCrudController | Thelia 2 API</title>

            <link rel="stylesheet" type="text/css" href="../../../css/bootstrap.min.css">
        <link rel="stylesheet" type="text/css" href="../../../css/bootstrap-theme.min.css">
        <link rel="stylesheet" type="text/css" href="../../../css/sami.css">

        <meta name="MobileOptimized" content="width">
        <meta name="HandheldFriendly" content="true">
        <meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1">

                <form id="search-form" action="../../../search.html" method="GET">

            <input name="search"
                   class="typeahead form-control"
                   type="search"
                   placeholder="Search">
        </form>

                <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-elements">
                    Toggle navigation

                </button>
                Thelia 2 API

Classes

Namespaces

Interfaces

Traits

Index

Search

class

Thelia

Controller

Admin

AbstractSeoCrudController

AbstractSeoCrudController

    class
    AbstractSeoCrudController        extends AbstractCrudController

Extend abstract CRUD controller to manage basic CRUD + SEO operations on a given object.

Traits

                                            Symfony\Component\DependencyInjection\ContainerAwareTrait

Constants

                EMPTY_FORM_NAME

                CONTROLLER_TYPE

                TEMPLATE_404

Methods

    EventDispatcher

                    getDispatcher()

Return the event dispatcher,

from
    BaseController

    Translator

                    getTranslator()

return the Translator

from
    BaseController

    MailerFactory

                    getMailer()

return an instance of \Swift_Mailer with good Transporter configured.

from
    BaseController

    BaseForm

                    createForm($name, $type = 'form',
    array $data = array(),
    array $options = array())

No description

from
    BaseController

    ContainerInterface

                    getContainer()

No description

from
    BaseController

    string

                    getControllerType()

No description

from
    BaseAdminController

                    adminLogAppend($resource, $action, $message, $resourceId = null)

Helper to append a message to the admin log.

from
    BaseAdminController

    Response

                    processTemplateAction(
    string $template)

This method process the rendering of view called from an admin page

from
    BaseAdminController

                    __construct(
    string $objectName,
    string $defaultListOrder,
    string $orderRequestParameterName,
    string $resourceCode,
    string $createEventIdentifier,
    string $updateEventIdentifier,
    string $deleteEventIdentifier,
    string $visibilityToggleEventIdentifier = null,
    string $changePositionEventIdentifier = null,
    string $updateSeoEventIdentifier = null,
    string $moduleCode = null)

No description

    Response

                    defaultAction()

The default action is displaying the list.

from
    AbstractCrudController

    Response

                    createAction()

Create a new object

from
    AbstractCrudController

    Response

                    updateAction()

Load a object for modification, and display the edit template.

from
    AbstractCrudController

    Response

                    processUpdateAction()

Save changes on a modified object, and either go back to the object list, or stay on the edition page.

from
    AbstractCrudController

                    updatePositionAction()

Update object position (only for objects whichsupport that)

from
    AbstractCrudController

                    setToggleVisibilityAction()

Online status toggle (only for object which support it)

from
    AbstractCrudController

    Response

                    deleteAction()

Delete an object

from
    AbstractCrudController

    Response

                    processUpdateSeoAction()

Update SEO modification, and either go back to the object list, or stay on the edition page.

Details

in
    BaseController at line 128

    EventDispatcher
    getDispatcher()

Return the event dispatcher,

Return Value

    EventDispatcher

in
    BaseController at line 138

    Translator
    getTranslator()

return the Translator

Return Value

    Translator

in
    BaseController at line 545

    MailerFactory
    getMailer()

return an instance of \Swift_Mailer with good Transporter configured.

Return Value

    MailerFactory

in
    BaseController at line 569

    BaseForm
    createForm($name, $type = 'form',
    array $data = array(),
    array $options = array())

Parameters

                $name

                $type

    array
                $data

    array
                $options

Return Value

    BaseForm
            This method builds a thelia form with its name

in
    BaseController at line 589

    ContainerInterface
    getContainer()

Return Value

    ContainerInterface

in
    BaseAdminController at line 90

    string
    getControllerType()

Return Value

    string

in
    BaseAdminController at line 52

    adminLogAppend($resource, $action, $message, $resourceId = null)

Helper to append a message to the admin log.

Parameters

                $resource

                $action

                $message

                $resourceId

in
    BaseAdminController at line 71

    Response
    processTemplateAction(
    string $template)

This method process the rendering of view called from an admin page

Parameters

    string
                $template
                the template name

Return Value

    Response
            the response which contains the rendered view

at line 49

    __construct(
    string $objectName,
    string $defaultListOrder,
    string $orderRequestParameterName,
    string $resourceCode,
    string $createEventIdentifier,
    string $updateEventIdentifier,
    string $deleteEventIdentifier,
    string $visibilityToggleEventIdentifier = null,
    string $changePositionEventIdentifier = null,
    string $updateSeoEventIdentifier = null,
    string $moduleCode = null)

Parameters

    string
                $objectName
                the lower case object name. Example. "message"

    string
                $defaultListOrder
                the default object list order, or null if list is not sortable. Example: manual

    string
                $orderRequestParameterName
                Name of the request parameter that set the list order (null if list is not sortable)

    string
                $resourceCode
                the 'resource' code. Example: "admin.configuration.message"

    string
                $createEventIdentifier
                the dispatched create TheliaEvent identifier. Example: TheliaEvents::MESSAGE_CREATE

    string
                $updateEventIdentifier
                the dispatched update TheliaEvent identifier. Example: TheliaEvents::MESSAGE_UPDATE

    string
                $deleteEventIdentifier
                the dispatched delete TheliaEvent identifier. Example: TheliaEvents::MESSAGE_DELETE

    string
                $visibilityToggleEventIdentifier
                the dispatched visibility toggle TheliaEvent identifier, or null if the object has no visible options. Example: TheliaEvents::MESSAGETOGGLEVISIBILITY

    string
                $changePositionEventIdentifier
                the dispatched position change TheliaEvent identifier, or null if the object has no position. Example: TheliaEvents::MESSAGEUPDATEPOSITION

    string
                $updateSeoEventIdentifier
                the dispatched update SEO change TheliaEvent identifier, or null if the object has no SEO. Example: TheliaEvents::MESSAGEUPDATESEO

    string
                $moduleCode
                The module code for ACL

in
    AbstractCrudController at line 290

    Response
    defaultAction()

The default action is displaying the list.

Return Value

    Response
            the response

in
    AbstractCrudController at line 305

    Response
    createAction()

Create a new object

Return Value

    Response
            the response

in
    AbstractCrudController at line 392

    Response
    updateAction()

Load a object for modification, and display the edit template.

Return Value

    Response
            the response

in
    AbstractCrudController at line 417

    Response
    processUpdateAction()

Save changes on a modified object, and either go back to the object list, or stay on the edition page.

Return Value

    Response
            the response

in
    AbstractCrudController at line 506

    updatePositionAction()

Update object position (only for objects whichsupport that)

in
    AbstractCrudController at line 581

    setToggleVisibilityAction()

Online status toggle (only for object which support it)

in
    AbstractCrudController at line 605

    Response
    deleteAction()

Delete an object

Return Value

    Response
            the response

at line 149

    Response
    processUpdateSeoAction()

Update SEO modification, and either go back to the object list, or stay on the edition page.

Return Value

    Response
            the response

        Generated by Sami, the API Documentation Generator.


text:
from twisted.internet import reactor
from twisted.web import client, error, http
from twisted.web.resource import Resource
from hookah import queue
import sys, os

import base64

from hookah import HookahRequest
from hookah import storage, queue

# TODO: Make these configurable
RETRIES = 3
DELAY_MULTIPLIER = 5

def dispatch_request(request):
    key = storage.instance.put(request)
    queue.instance.submit(key)

def post_and_retry(url, data, retry=0, content_type='application/x-www-form-urlencoded'):
    if type(data) is dict:
        print "Posting [%s] to %s with %s" % (retry, url, data)
        data = urllib.urlencode(data)
    else:
        print "Posting [%s] to %s with %s bytes of postdata" % (retry, url, len(data))
    headers = {
        'Content-Type': content_type,
        'Content-Length': str(len(data)),
    }
    client.getPage(url, method='POST' if len(data) else 'GET', headers=headers, postdata=data if len(data) else None).addCallbacks( \
                    if_success, lambda reason: if_fail(reason, url, data, retry, content_type))

def if_success(page): pass
def if_fail(reason, url, data, retry, content_type):
    if reason.getErrorMessage()[0:3] in ['301', '302', '303']:
        return # Not really a fail
    print reason.getErrorMessage()
    if retry < RETRIES:
        retry += 1
        reactor.callLater(retry * DELAY_MULTIPLIER, post_and_retry, url, data, retry, content_type)

class DispatchResource(Resource):
    isLeaf = True

    def render(self, request):
        url = base64.b64decode(request.postpath[0])

        if url:
            headers = {}
            for header in ['content-type', 'content-length']:
                value = request.getHeader(header)
                if value:
                    headers[header] = value

            dispatch_request(HookahRequest(url, headers, request.content.read()))

            request.setResponseCode(http.ACCEPTED)
            return "202 Scheduled"
        else:
            request.setResponseCode(http.BAD_REQUEST)
            return "400 No destination URL"

if __name__ == '__main__':
    from twisted.web.server import Request
    from cStringIO import StringIO
    class TestRequest(Request):
        postpath = ['aHR0cDovL3Byb2dyaXVtLmNvbT9ibGFo']
        content = StringIO("BLAH")

    output = DispatchResource().render(TestRequest({}, True))
    print output
    assert output == 'BLAH'


text:
// Copyright (c) 2013, Webit Team. All Rights Reserved.
package webit.schedule;

/**
 *
 * @author zqq90
 */
public interface TaskExecutor {

    int STATE_STOPPED = 1;
    int STATE_PAUSED = 2;
    int STATE_STOPPING = 3;
    int STATE_PAUSING = 4;
    int STATE_RUNNING = 5;
    int STATE_HOLDING = 6;

    void runIfNot(Time time);

    void stopAndWait();

    void goonIfPaused();

    void askforPause();

    void askforStop();

    Task getTask();

    Scheduler getScheduler();

    int getState();
}


text:
Assignment: LL and SC
=====================

Synchronization and Interprocess Communication
----------------------------------------------

This assignment is a Xinu assignment allowing the student to grow a
more firm understanding of how an operating system works. This
assignment is part of the :doc:`/teaching/Student-Built-Xinu` track for
professors that are :doc:`teaching with Xinu </teaching/index>`. Written answers
to the analysis questions should be put into a file called
``system/ANALYSIS.txt`` and submitted with the code. The entire
directory containing the operating system should be turned in during
the submission process.

This assignment includes a LAB REPORT. Include your lab report in a file
called ``login.pdf``, (with your login name,) in the top-level directory
of your xinu-hw6.

After completion of this assignment we will have a basic operating
system with preemptive, priority scheduling of processes, counting
semaphores with wait queues as synchronization primitives and
interprocess communication with bounded buffers.

Atomicity
---------

In a system with preemption, we must now guard against being interrupted
while in the middle of an atomic operation. There are many points in
operating system code where an unfortunately timed interrupt could leave
the system in an inconsistent state.

For example, in the ``resched()`` function, there are several lines of C
code (which translate into many more lines of machine code) between the
point where the outgoing running process is put back into the queue of
ready processes and the point where an incoming process is dequeued and
set to running. If an interrupt were to take place in the midst of this
transition, the interrupt handling code might see the system in an
inconsistent state, in which there doesn't seem to be a currently
running process. For this reason, we consider process rescheduling to be
atomic with respect to the rest of the operating system. The easiest way
to enforce this is to temporarily disable interrupts during a critical,
atomic section of code, and then reenable them when done.

The new file ``system/intr.S`` contains functions ``enable()``,
``disable()``, and ``restore()`` for manipulating the master bit that
enables processor interrupts. From this point forward, these functions
must be used judiciously to guard atomic sections in the operating
system functions you write.

Preparation
-----------

First, make a fresh copy of your work thus far::

  cp -R

Untar the new project files on top of this new directory::

  tar xvzf

You should now see the new project files in with your old files. Be
certain to make clean before compiling for the first time.

Semaphores
----------

With this assignment you must learn about and understand classic
semaphores before using them to implement the remainder of the
assignment. An implementation of classic semaphores with waiting queues
has been provided for you. Please examine and understand the
implementation which can be found across several files including
``include/semaphore.h``, ``system/newsem.c``, system/signal.c and
``system/wait.c``. As part of the analysis portion of this assignment,
you will need to write a main file to show your understanding of
semaphores.

Producers/Consumers
-------------------

Using the provided semaphore structure, implement producers and
consumers that communicate using a Bounded-Buffer. Your textbook
provides discussion of the Bounded-Buffer Problem beginning in
section 6.6.1, and outlines this assignment as
Programming Project 6.40 - Producer-Consumer Problem.

Segments of the textbook code are already given for you in
``system/testcases.c``. Complete the project as specified in the text
with slight adjustments as noted in the testcases TODOs. Answer the
analysis questions below in your lab report.

Synchronization Hardware
------------------------

Disabling all interrupts is an effective but heavy-handed approach for
providing mutual exclusion. Multicore systems and complex real-time
systems often cannot afford to disable interrupts, and rely more on
hardware support for atomic updates.

The assembly code in ``system/testAndSet.S`` implements the
``testAndSet()`` operation (ala Figure 6.4 in your textbook [1]_) using
the MIPS LL/SC (load-linked and store-conditional) opcodes. Much more
complex synchronization primitives can be built on top of simple atomic
opcode combinations such as there.

Build ``mutexAcquire()`` and ``mutexRelease()`` in ``system/mutex.c``
using the bounded-waiting algorithm presented in Figure 6.8 of your
textbook [1]_.

Lab Report
----------

For the lab report portion of this assignment, you should typeset a
document containing your analysis of the system components introduced. A
sample report format is given
`HERE <http://xinu/wiki/images/b/b6/Labreport.pdf>`__. We expect the
report to be written with your exemplary literacy skills.

What are we reporting on? Once you have the Producer/Consumer
implementation working with the provided semaphore API, consider the
following questions:

#. Does it matter which process runs first, the producer or the
   consumer?
#. What happens when there are multiple producers and consumers?
#. What happens when the producer(s) priority is higher than the
   consumer(s)? Vice-versa?

For each of these, form a hypothesis, construct an experiment
(testcases), and draw appropriate conclusions. There is no benefit in
fudging your hypotheses after you know the answer; the quality of a lab
report is orthogonal to whether your hypothesis was correct or not.

For part two of the assignment, use your bounded-wait ``mutexAcquire()``
and ``mutexRelease()`` to prevent your producers/consumers from breaking
up ``kprintf()`` calls mid-line without additional disabling of
interrupts. Consider the following questions:

#. Can your bounded-wait mutex subsystem replace the semaphore subsystem
   for this task?
#. Under what conditions in the embedded operating system will the mutex
   subsystem not work as designed?
#. Can you deadlock your producer and consumer?

I rewrote the ``testAndSet()`` MIPS code from lab today to correspond
more closely to the semantics used in the textbook chapter 6 [1]_. There
is an example of the `lab report
format <http://xinu/wiki/images/b/b6/Labreport.pdf>`__, and analysis
questions to report upon in the final section. Typeset your lab report
with whatever software you are comfortable with.

References
----------

.. [1]
   Silberschatz, A., Galvin, P. B., and Gagne, G. 2009 Operating System
   Concepts. 8th. John Wiley & Sons, Inc.


text:
namespace YAML
{
	enum EMITTER_MANIP {
		// general manipulators
		Auto,
		TagByKind,
		Newline,

		// output character set
		EmitNonAscii,
		EscapeNonAscii,

		// string manipulators
		// Auto, // duplicate
		SingleQuoted,
		DoubleQuoted,
		Literal,

		// bool manipulators
		YesNoBool,  // yes, no
		TrueFalseBool,  // true, false
		OnOffBool,  // on, off
		UpperCase,  // TRUE, N
		LowerCase,  // f, yes
		CamelCase,  // No, Off
		LongBool,  // yes, On
		ShortBool,  // y, t

		// int manipulators
		Dec,
		Hex,
		Oct,

		// document manipulators
		BeginDoc,
		EndDoc,

		// sequence manipulators
		BeginSeq,
		EndSeq,
		Flow,
		Block,

		// map manipulators
		BeginMap,
		EndMap,
		Key,
		Value,
		// Flow, // duplicate
		// Block, // duplicate
		// Auto, // duplicate
		LongKey
	};

	struct _Indent {
		_Indent(int value_): value(value_) {}
		int value;
	};

	inline _Indent Indent(int value) {
		return _Indent(value);
	}

	struct _Alias {
		_Alias(const std::string& content_): content(content_) {}
		std::string content;
	};

	inline _Alias Alias(const std::string content) {
		return _Alias(content);
	}

	struct _Anchor {
		_Anchor(const std::string& content_): content(content_) {}
		std::string content;
	};

	inline _Anchor Anchor(const std::string content) {
		return _Anchor(content);
	}

	struct _Tag {
		struct Type { enum value { Verbatim, PrimaryHandle, NamedHandle }; };

		explicit _Tag(const std::string& prefix_, const std::string& content_, Type::value type_)
		: prefix(prefix_), content(content_), type(type_)
		{
		}
		std::string prefix;
		std::string content;
		Type::value type;
	};

	inline _Tag VerbatimTag(const std::string content) {
		return _Tag("", content, _Tag::Type::Verbatim);
	}

	inline _Tag LocalTag(const std::string content) {
		return _Tag("", content, _Tag::Type::PrimaryHandle);
	}

	inline _Tag LocalTag(const std::string& prefix, const std::string content) {
		return _Tag(prefix, content, _Tag::Type::NamedHandle);
	}

	inline _Tag SecondaryTag(const std::string content) {
		return _Tag("", content, _Tag::Type::NamedHandle);
	}

	struct _Comment {
		_Comment(const std::string& content_): content(content_) {}
		std::string content;
	};

	inline _Comment Comment(const std::string content) {
		return _Comment(content);
	}

    struct _Precision {
        _Precision(int floatPrecision_, int doublePrecision_): floatPrecision(floatPrecision_), doublePrecision(doublePrecision_) {}

        int floatPrecision;
        int doublePrecision;
    };

    inline _Precision FloatPrecision(int n) {
        return _Precision(n, -1);
    }

    inline _Precision DoublePrecision(int n) {
        return _Precision(-1, n);
    }

    inline _Precision Precision(int n) {
        return _Precision(n, n);
    }
}

#endif // EMITTERMANIP_H_62B23520_7C8E_11DE_8A39_0800200C9A66


text:
using Proto.Router.Messages;
using Proto.Router.Routers;
using Proto.Mailbox;

namespace Proto.Router
{
    public class RouterProcess : LocalProcess
    {
        private readonly RouterState _state;

        public RouterProcess(RouterState state, IMailbox mailbox) : base(mailbox)
        {
            _state = state;
        }

        protected override void SendUserMessage(PID pid, object message)
        {
            var (msg,_,_) = MessageEnvelope.Unwrap(message);
            switch (msg)
            {
                case RouterManagementMessage _:
                    base.SendUserMessage(pid,message);
                    break;
                default:
                    _state.RouteMessage(message);
                    break;
            }
        }
    }
}


text:
from __future__ import absolute_import
from systrace.tracing_agents import atrace_agent

class AtraceConfig():
  """Stores configuration options specific to Atrace.

    categories: List that specifies the Atrace categories to trace.
        Example: ['sched', 'webview']
    app_name: String or list that specifies the application name (or names)
        on which to run application level tracing.
        Example: 'org.chromium.webview_shell'
  """
  def __init__(self):
    self.categories = atrace_agent.DEFAULT_CATEGORIES
    self.app_name = None


text:
<?xml version="1.0" encoding="UTF-8"?>
<features
        name="cxfatwork-repo-${project.version}"

        xmlns="http://karaf.apache.org/xmlns/features/v1.0.0"
        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
        xsi:schemaLocation="http://karaf.apache.org/xmlns/features/v1.0.0 http://karaf.apache.org/xmlns/features/v1.0.0">

    <repository>mvn:org.apache.cxf.karaf/apache-cxf/${cxf.version}/xml/features</repository>
    <!--
    <repository>mvn:org.apache.cxf.dosgi/cxf-dosgi/${cxf.dosgi.version}/xml/features</repository>
    -->

    <!-- This feature is used to install all the needed modules/features for the REST API to run -->
    <feature name="cxfatwork-environment"
             description="Repository to enable CXF-RS Rest services"
             version="${project.version}">

        <!-- Karaf feature dependencies -->

        <feature>jndi</feature>
        <feature>transaction</feature>
        <feature>jpa</feature>
        <feature>webconsole</feature>
        <feature>spring/3.1.4.RELEASE</feature>

        <!-- Mixed third-party bundles -->
        <bundle>mvn:org.apache.xbean/xbean-blueprint/${xbean-blueprint.version}</bundle>

        <bundle>mvn:org.apache.aries.transaction/org.apache.aries.transaction.jdbc/${apache-aries-transaction}</bundle>
        <bundle>mvn:org.apache.geronimo.specs/geronimo-j2ee-connector_1.5_spec/${geronimo-j2ee-connector_1.5_spec.version}</bundle>
        <bundle>mvn:org.apache.geronimo.components/geronimo-transaction/${geronimo-transaction.version}</bundle>
        <bundle>mvn:org.apache.geronimo.components/geronimo-connector/${geronimo-connector.version}</bundle>

        <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.commons-collections/${smx-bundles-commons-collections.version}</bundle>
        <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.commons-dbcp/${smx-bundles-commons-dbcp.version}</bundle>
        <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.commons-lang/${smx-bundles-commons-lang.version}</bundle>
        <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.commons-pool/${smx-bundles-commons-pool.version}</bundle>

        <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.serp/${smx-bundles-serp.version}</bundle>
        <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.neethi/${smx-bundles-nethi.version}</bundle>
        <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.reflections/${smx-bundles-reflections.version}</bundle>

        <bundle>mvn:org.apache.servicemix.specs/org.apache.servicemix.specs.stax-api-1.2/${smx-specs-stax-api-1.2.version}</bundle>

        <!-- -->
        <bundle>mvn:javax.jws/com.springsource.javax.jws/2.0.0</bundle>
        <bundle>mvn:com.google.guava/guava/${guava.version}</bundle>

        <!-- Jackson Modules Installation -->
        <bundle>mvn:org.codehaus.jackson/jackson-core-asl/${codehaus-jackson.version}</bundle>
        <bundle>mvn:org.codehaus.jackson/jackson-mapper-asl/${codehaus-jackson.version}</bundle>
        <bundle>mvn:org.codehaus.jackson/jackson-jaxrs/${codehaus-jackson.version}</bundle>

        <!-- SWAGGER DEPENDENCIES AS IN SERVICEMIX - START -->
        <bundle>mvn:org.scala-lang/scala-library/2.10.2</bundle>

        <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.swagger-jaxrs/${smx-bundles-swagger-jaxrs.version}</bundle>
        <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.json4s/${smx-bundles-json4s.version}</bundle>
        <!--<bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.reflections/0.9.8_1</bundle>-->
        <!--<bundle>mvn:com.google.guava/guava/14.0.1</bundle>-->

        <bundle>mvn:com.fasterxml.jackson.core/jackson-annotations/${fasterxml-jackson.version}</bundle>
        <bundle>mvn:com.fasterxml.jackson.core/jackson-databind/${fasterxml-jackson.version}</bundle>
        <bundle>mvn:com.fasterxml.jackson.core/jackson-core/${fasterxml-jackson.version}</bundle>
        <bundle>mvn:com.fasterxml.jackson.jaxrs/jackson-jaxrs-json-provider/${fasterxml-jackson.version}</bundle>
        <bundle>mvn:com.fasterxml.jackson.jaxrs/jackson-jaxrs-base/${fasterxml-jackson.version}</bundle>
        <bundle>mvn:com.fasterxml.jackson.module/jackson-module-jaxb-annotations/${fasterxml-jackson.version}</bundle>

        <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.commons-codec/${smx-bundles-commons-codec.version}</bundle>
        <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.commons-httpclient/${smx-bundles-commons-httpclient.version}</bundle>
        <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.javassist/${smx-bundles-javassist.version}</bundle>
        <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.swagger-annotations/${smx-bundles-swagger-annotation.version}</bundle>
        <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.swagger-core/${smx-bundles-swagger-core.version}</bundle>
        <bundle>mvn:org.apache.servicemix.bundles/org.apache.servicemix.bundles.jackson-module-scala/${smx-bundles-jackson-scala.version}</bundle>
        <!-- SWAGGER DEPENDENCIES AS IN SERVICEMIX - END -->

        <!-- CXF installation -->
        <feature>cxf</feature>
        <feature>cxf-tools</feature>
        <!--
        <feature>cxf-dosgi-core</feature>
        <feature>cxf-dosgi-discovery-local</feature>
        <feature>cxf-dosgi-discovery-distributed</feature>
        -->
    </feature>

    <feature name="cxfatwork-rest-osgi-modules"
             description="Repository to install the Tutorials for CXF-RS"
             version="${project.version}">

        <feature>cxfatwork-environment</feature>

        <bundle>mvn:name.marmac.tutorials/cxfatwork-rs-osgi-resources/${project.version}</bundle>
        <bundle>mvn:name.marmac.tutorials/cxfatwork-rs-osgi-api/${project.version}</bundle>
        <bundle>mvn:name.marmac.tutorials/cxfatwork-rs-osgi-impl-jaxrs/${project.version}</bundle>

        <configfile
                override="true"
                finalname="/etc/name.marmac.tutorials.cxfatwork.cxfrsosgijaxrs.cfg">mvn:name.marmac.tutorials/cxfatwork-rs-osgi-impl-jaxrs/${project.version}/cfg</configfile>

    </feature>

</features>