text:
package org.apache.geode.internal.offheap;

import static org.junit.Assert.*;

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.concurrent.atomic.AtomicReference;

import org.junit.Rule;
import org.junit.Test;
import org.junit.contrib.java.lang.system.RestoreSystemProperties;
import org.junit.experimental.categories.Category;

import org.apache.geode.OutOfOffHeapMemoryException;
import org.apache.geode.cache.CacheClosedException;
import org.apache.geode.test.junit.categories.UnitTest;

@Category(UnitTest.class)
public class MemoryAllocatorJUnitTest {

  private long expectedMemoryUsage;
  private boolean memoryUsageEventReceived;

  @Rule
  public final RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties();

  private static int round(int multiple, int v) {
    return ((v + multiple - 1) / multiple) * multiple;
  }

  @Test
  public void testNullGetAllocator() {
    try {
      MemoryAllocatorImpl.getAllocator();
      fail("expected CacheClosedException");
    } catch (CacheClosedException expected) {
    }
  }

  @Test
  public void testConstructor() {
    try {
      MemoryAllocatorImpl.createForUnitTest(null, null, null);
      fail("expected IllegalArgumentException");
    } catch (IllegalArgumentException expected) {
    }
  }

  @Test
  public void testCreate() {
    System.setProperty(MemoryAllocatorImpl.FREE_OFF_HEAP_MEMORY_PROPERTY, "false");
    {
      NullOutOfOffHeapMemoryListener listener = new NullOutOfOffHeapMemoryListener();
      NullOffHeapMemoryStats stats = new NullOffHeapMemoryStats();
      try {
        MemoryAllocatorImpl.createForUnitTest(listener, stats, 10, 950, 100, new SlabFactory() {
          @Override
          public Slab create(int size) {
            throw new OutOfMemoryError("expected");
          }
        });
      } catch (OutOfMemoryError expected) {
      }
      assertTrue(listener.isClosed());
      assertTrue(stats.isClosed());
    }
    {
      NullOutOfOffHeapMemoryListener listener = new NullOutOfOffHeapMemoryListener();
      NullOffHeapMemoryStats stats = new NullOffHeapMemoryStats();
      int MAX_SLAB_SIZE = 100;
      try {
        SlabFactory factory = new SlabFactory() {
          private int createCount = 0;

          @Override
          public Slab create(int size) {
            createCount++;
            if (createCount == 1) {
              return new SlabImpl(size);
            } else {
              throw new OutOfMemoryError("expected");
            }
          }
        };
        MemoryAllocatorImpl.createForUnitTest(listener, stats, 10, 950, MAX_SLAB_SIZE, factory);
      } catch (OutOfMemoryError expected) {
      }
      assertTrue(listener.isClosed());
      assertTrue(stats.isClosed());
    }
    {
      NullOutOfOffHeapMemoryListener listener = new NullOutOfOffHeapMemoryListener();
      NullOffHeapMemoryStats stats = new NullOffHeapMemoryStats();
      SlabFactory factory = new SlabFactory() {
        @Override
        public Slab create(int size) {
          return new SlabImpl(size);
        }
      };
      MemoryAllocator ma =
          MemoryAllocatorImpl.createForUnitTest(listener, stats, 10, 950, 100, factory);
      try {
        assertFalse(listener.isClosed());
        assertFalse(stats.isClosed());
        ma.close();
        assertTrue(listener.isClosed());
        assertFalse(stats.isClosed());
        listener = new NullOutOfOffHeapMemoryListener();
        NullOffHeapMemoryStats stats2 = new NullOffHeapMemoryStats();
        {
          SlabImpl slab = new SlabImpl(1024);
          try {
            MemoryAllocatorImpl.createForUnitTest(listener, stats2, new SlabImpl[] {slab});
          } catch (IllegalStateException expected) {
            assertTrue("unexpected message: " + expected.getMessage(), expected.getMessage().equals(
                "attempted to reuse existing off-heap memory even though new off-heap memory was allocated"));
          } finally {
            slab.free();
          }
          assertFalse(stats.isClosed());
          assertTrue(listener.isClosed());
          assertTrue(stats2.isClosed());
        }
        listener = new NullOutOfOffHeapMemoryListener();
        stats2 = new NullOffHeapMemoryStats();
        MemoryAllocator ma2 =
            MemoryAllocatorImpl.createForUnitTest(listener, stats2, 10, 950, 100, factory);
        assertSame(ma, ma2);
        assertTrue(stats.isClosed());
        assertFalse(listener.isClosed());
        assertFalse(stats2.isClosed());
        stats = stats2;
        ma.close();
        assertTrue(listener.isClosed());
        assertFalse(stats.isClosed());
      } finally {
        MemoryAllocatorImpl.freeOffHeapMemory();
      }
      assertTrue(stats.isClosed());
    }
  }

  @Test
  public void testBasics() {
    int BATCH_SIZE = 1;
    int TINY_MULTIPLE = FreeListManager.TINY_MULTIPLE;
    int HUGE_MULTIPLE = FreeListManager.HUGE_MULTIPLE;
    int perObjectOverhead = OffHeapStoredObject.HEADER_SIZE;
    int maxTiny = FreeListManager.MAX_TINY - perObjectOverhead;
    int minHuge = maxTiny + 1;
    int TOTAL_MEM = (maxTiny + perObjectOverhead) * BATCH_SIZE
        /* + (maxBig+perObjectOverhead)*BATCH_SIZE */ + round(TINY_MULTIPLE,
            minHuge + 1 + perObjectOverhead) * BATCH_SIZE
        + (TINY_MULTIPLE + perObjectOverhead)
            * BATCH_SIZE /* + (MIN_BIG_SIZE+perObjectOverhead)*BATCH_SIZE */
        + round(TINY_MULTIPLE, minHuge + perObjectOverhead + 1);
    SlabImpl slab = new SlabImpl(TOTAL_MEM);
    try {
      MemoryAllocatorImpl ma =
          MemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(),
              new NullOffHeapMemoryStats(), new SlabImpl[] {slab});
      assertEquals(TOTAL_MEM, ma.getFreeMemory());
      assertEquals(TOTAL_MEM, ma.freeList.getFreeFragmentMemory());
      assertEquals(0, ma.freeList.getFreeTinyMemory());
      assertEquals(0, ma.freeList.getFreeHugeMemory());
      StoredObject tinymc = ma.allocate(maxTiny);
      assertEquals(TOTAL_MEM - round(TINY_MULTIPLE, maxTiny + perObjectOverhead),
          ma.getFreeMemory());
      assertEquals(round(TINY_MULTIPLE, maxTiny + perObjectOverhead) * (BATCH_SIZE - 1),
          ma.freeList.getFreeTinyMemory());
      StoredObject hugemc = ma.allocate(minHuge);
      assertEquals(TOTAL_MEM
          - round(TINY_MULTIPLE,
              minHuge + perObjectOverhead)/*-round(BIG_MULTIPLE, maxBig+perObjectOverhead)*/
          - round(TINY_MULTIPLE, maxTiny + perObjectOverhead), ma.getFreeMemory());
      long freeSlab = ma.freeList.getFreeFragmentMemory();
      long oldFreeHugeMemory = ma.freeList.getFreeHugeMemory();
      assertEquals(round(TINY_MULTIPLE, minHuge + perObjectOverhead) * (BATCH_SIZE - 1),
          oldFreeHugeMemory);
      hugemc.release();
      assertEquals(round(TINY_MULTIPLE, minHuge + perObjectOverhead),
          ma.freeList.getFreeHugeMemory() - oldFreeHugeMemory);
      assertEquals(TOTAL_MEM/*-round(BIG_MULTIPLE, maxBig+perObjectOverhead)*/ - round(
          TINY_MULTIPLE, maxTiny + perObjectOverhead), ma.getFreeMemory());
      assertEquals(TOTAL_MEM - round(TINY_MULTIPLE, maxTiny + perObjectOverhead),
          ma.getFreeMemory());
      long oldFreeTinyMemory = ma.freeList.getFreeTinyMemory();
      tinymc.release();
      assertEquals(round(TINY_MULTIPLE, maxTiny + perObjectOverhead),
          ma.freeList.getFreeTinyMemory() - oldFreeTinyMemory);
      assertEquals(TOTAL_MEM, ma.getFreeMemory());
      // now lets reallocate from the free lists
      tinymc = ma.allocate(maxTiny);
      assertEquals(oldFreeTinyMemory, ma.freeList.getFreeTinyMemory());
      assertEquals(TOTAL_MEM - round(TINY_MULTIPLE, maxTiny + perObjectOverhead),
          ma.getFreeMemory());
      hugemc = ma.allocate(minHuge);
      assertEquals(oldFreeHugeMemory, ma.freeList.getFreeHugeMemory());
      assertEquals(TOTAL_MEM
          - round(TINY_MULTIPLE,
              minHuge + perObjectOverhead)/*-round(BIG_MULTIPLE, maxBig+perObjectOverhead)*/
          - round(TINY_MULTIPLE, maxTiny + perObjectOverhead), ma.getFreeMemory());
      hugemc.release();
      assertEquals(round(TINY_MULTIPLE, minHuge + perObjectOverhead),
          ma.freeList.getFreeHugeMemory() - oldFreeHugeMemory);
      assertEquals(TOTAL_MEM/*-round(BIG_MULTIPLE, maxBig+perObjectOverhead)*/ - round(
          TINY_MULTIPLE, maxTiny + perObjectOverhead), ma.getFreeMemory());
      assertEquals(TOTAL_MEM - round(TINY_MULTIPLE, maxTiny + perObjectOverhead),
          ma.getFreeMemory());
      tinymc.release();
      assertEquals(round(TINY_MULTIPLE, maxTiny + perObjectOverhead),
          ma.freeList.getFreeTinyMemory() - oldFreeTinyMemory);
      assertEquals(TOTAL_MEM, ma.getFreeMemory());
      // None of the reallocates should have come from the slab.
      assertEquals(freeSlab, ma.freeList.getFreeFragmentMemory());
      tinymc = ma.allocate(1);
      assertEquals(round(TINY_MULTIPLE, 1 + perObjectOverhead), tinymc.getSize());
      assertEquals(freeSlab - (round(TINY_MULTIPLE, 1 + perObjectOverhead) * BATCH_SIZE),
          ma.freeList.getFreeFragmentMemory());
      freeSlab = ma.freeList.getFreeFragmentMemory();
      tinymc.release();
      assertEquals(
          round(TINY_MULTIPLE, maxTiny + perObjectOverhead)
              + (round(TINY_MULTIPLE, 1 + perObjectOverhead) * BATCH_SIZE),
          ma.freeList.getFreeTinyMemory() - oldFreeTinyMemory);

      hugemc = ma.allocate(minHuge + 1);
      assertEquals(round(TINY_MULTIPLE, minHuge + 1 + perObjectOverhead), hugemc.getSize());
      assertEquals(round(TINY_MULTIPLE, minHuge + perObjectOverhead) * (BATCH_SIZE - 1),
          ma.freeList.getFreeHugeMemory());
      hugemc.release();
      assertEquals(round(TINY_MULTIPLE, minHuge + perObjectOverhead) * BATCH_SIZE,
          ma.freeList.getFreeHugeMemory());
      hugemc = ma.allocate(minHuge);
      assertEquals(round(TINY_MULTIPLE, minHuge + perObjectOverhead) * (BATCH_SIZE - 1),
          ma.freeList.getFreeHugeMemory());
      if (BATCH_SIZE > 1) {
        StoredObject hugemc2 = ma.allocate(minHuge);
        assertEquals(round(TINY_MULTIPLE, minHuge + perObjectOverhead) * (BATCH_SIZE - 2),
            ma.freeList.getFreeHugeMemory());
        hugemc2.release();
        assertEquals(round(TINY_MULTIPLE, minHuge + perObjectOverhead) * (BATCH_SIZE - 1),
            ma.freeList.getFreeHugeMemory());
      }
      hugemc.release();
      assertEquals(round(TINY_MULTIPLE, minHuge + perObjectOverhead) * BATCH_SIZE,
          ma.freeList.getFreeHugeMemory());
      // now that we do defragmentation the following allocate works.
      hugemc = ma.allocate(minHuge + HUGE_MULTIPLE + HUGE_MULTIPLE - 1);
    } finally {
      MemoryAllocatorImpl.freeOffHeapMemory();
    }
  }

  @Test
  public void testChunkCreateDirectByteBuffer() {
    SlabImpl slab = new SlabImpl(1024 * 1024);
    try {
      MemoryAllocatorImpl ma =
          MemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(),
              new NullOffHeapMemoryStats(), new SlabImpl[] {slab});
      ByteBuffer bb = ByteBuffer.allocate(1024);
      for (int i = 0; i < 1024; i++) {
        bb.put((byte) i);
      }
      bb.position(0);
      OffHeapStoredObject c =
          (OffHeapStoredObject) ma.allocateAndInitialize(bb.array(), false, false);
      assertEquals(1024, c.getDataSize());
      if (!Arrays.equals(bb.array(), c.getRawBytes())) {
        fail("arrays are not equal. Expected " + Arrays.toString(bb.array()) + " but found: "
            + Arrays.toString(c.getRawBytes()));
      }
      ByteBuffer dbb = c.createDirectByteBuffer();
      assertEquals(true, dbb.isDirect());
      assertEquals(bb, dbb);
    } finally {
      MemoryAllocatorImpl.freeOffHeapMemory();
    }
  }

  @Test
  public void testDebugLog() {
    MemoryAllocatorImpl.debugLog("test debug log", false);
    MemoryAllocatorImpl.debugLog("test debug log", true);
  }

  @Test
  public void testGetLostChunks() {
    SlabImpl slab = new SlabImpl(1024 * 1024);
    try {
      MemoryAllocatorImpl ma =
          MemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(),
              new NullOffHeapMemoryStats(), new SlabImpl[] {slab});
      assertEquals(Collections.emptyList(), ma.getLostChunks(null));
    } finally {
      MemoryAllocatorImpl.freeOffHeapMemory();
    }
  }

  @Test
  public void testFindSlab() {
    final int SLAB_SIZE = 1024 * 1024;
    SlabImpl slab = new SlabImpl(SLAB_SIZE);
    try {
      MemoryAllocatorImpl ma =
          MemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(),
              new NullOffHeapMemoryStats(), new SlabImpl[] {slab});
      assertEquals(0, ma.findSlab(slab.getMemoryAddress()));
      assertEquals(0, ma.findSlab(slab.getMemoryAddress() + SLAB_SIZE - 1));
      try {
        ma.findSlab(slab.getMemoryAddress() - 1);
        fail("expected IllegalStateException");
      } catch (IllegalStateException expected) {
      }
      try {
        ma.findSlab(slab.getMemoryAddress() + SLAB_SIZE);
        fail("expected IllegalStateException");
      } catch (IllegalStateException expected) {
      }
    } finally {
      MemoryAllocatorImpl.freeOffHeapMemory();
    }
  }

  @Test
  public void testValidateAddressAndSize() {
    final int SLAB_SIZE = 1024 * 1024;
    SlabImpl slab = new SlabImpl(SLAB_SIZE);
    try {
      MemoryAllocatorImpl ma =
          MemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(),
              new NullOffHeapMemoryStats(), new SlabImpl[] {slab});
      try {
        MemoryAllocatorImpl.validateAddress(0L);
        fail("expected IllegalStateException");
      } catch (IllegalStateException expected) {
        assertEquals("Unexpected exception message: " + expected.getMessage(), true,
            expected.getMessage().contains("addr was smaller than expected"));
      }
      try {
        MemoryAllocatorImpl.validateAddress(1L);
        fail("expected IllegalStateException");
      } catch (IllegalStateException expected) {
        assertEquals("Unexpected exception message: " + expected.getMessage(), true, expected
            .getMessage().contains("Valid addresses must be in one of the following ranges:"));
      }
      MemoryAllocatorImpl.validateAddressAndSizeWithinSlab(slab.getMemoryAddress(), SLAB_SIZE,
          false);
      MemoryAllocatorImpl.validateAddressAndSizeWithinSlab(slab.getMemoryAddress(), SLAB_SIZE,
          true);
      MemoryAllocatorImpl.validateAddressAndSizeWithinSlab(slab.getMemoryAddress(), -1, true);
      try {
        MemoryAllocatorImpl.validateAddressAndSizeWithinSlab(slab.getMemoryAddress() - 1, SLAB_SIZE,
            true);
        fail("expected IllegalStateException");
      } catch (IllegalStateException expected) {
        assertEquals("Unexpected exception message: " + expected.getMessage(), true,
            expected.getMessage()
                .equals(" address 0x" + Long.toString(slab.getMemoryAddress() - 1, 16)
                    + " does not address the original slab memory"));
      }
      try {
        MemoryAllocatorImpl.validateAddressAndSizeWithinSlab(slab.getMemoryAddress(), SLAB_SIZE + 1,
            true);
        fail("expected IllegalStateException");
      } catch (IllegalStateException expected) {
        assertEquals("Unexpected exception message: " + expected.getMessage(), true,
            expected.getMessage()
                .equals(" address 0x" + Long.toString(slab.getMemoryAddress() + SLAB_SIZE, 16)
                    + " does not address the original slab memory"));
      }
    } finally {
      MemoryAllocatorImpl.freeOffHeapMemory();
    }
  }

  @Test
  public void testMemoryInspection() {
    final int SLAB_SIZE = 1024 * 1024;
    SlabImpl slab = new SlabImpl(SLAB_SIZE);
    try {
      MemoryAllocatorImpl ma =
          MemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(),
              new NullOffHeapMemoryStats(), new SlabImpl[] {slab});
      MemoryInspector inspector = ma.getMemoryInspector();
      assertNotNull(inspector);
      assertEquals(null, inspector.getFirstBlock());
      assertEquals(Collections.emptyList(), inspector.getSnapshot());
      assertEquals(Collections.emptyList(), inspector.getAllocatedBlocks());
      assertEquals(null, inspector.getBlockAfter(null));
      inspector.createSnapshot();
      // call this twice for code coverage
      inspector.createSnapshot();
      try {
        assertEquals(inspector.getAllBlocks(), inspector.getSnapshot());
        MemoryBlock firstBlock = inspector.getFirstBlock();
        assertNotNull(firstBlock);
        assertEquals(1024 * 1024, firstBlock.getBlockSize());
        assertEquals("N/A", firstBlock.getDataType());
        assertEquals(-1, firstBlock.getFreeListId());
        assertTrue(firstBlock.getAddress() > 0);
        assertNull(firstBlock.getNextBlock());
        assertEquals(0, firstBlock.getRefCount());
        assertEquals(0, firstBlock.getSlabId());
        assertEquals(MemoryBlock.State.UNUSED, firstBlock.getState());
        assertFalse(firstBlock.isCompressed());
        assertFalse(firstBlock.isSerialized());
        assertEquals(null, inspector.getBlockAfter(firstBlock));
      } finally {
        inspector.clearSnapshot();
      }
    } finally {
      MemoryAllocatorImpl.freeOffHeapMemory();
    }
  }

  @Test
  public void testClose() {
    System.setProperty(MemoryAllocatorImpl.FREE_OFF_HEAP_MEMORY_PROPERTY, "false");
    SlabImpl slab = new SlabImpl(1024 * 1024);
    boolean freeSlab = true;
    SlabImpl[] slabs = new SlabImpl[] {slab};
    try {
      MemoryAllocatorImpl ma = MemoryAllocatorImpl.createForUnitTest(
          new NullOutOfOffHeapMemoryListener(), new NullOffHeapMemoryStats(), slabs);
      ma.close();
      ma.close();
      System.setProperty(MemoryAllocatorImpl.FREE_OFF_HEAP_MEMORY_PROPERTY, "true");
      try {
        ma = MemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(),
            new NullOffHeapMemoryStats(), slabs);
        ma.close();
        freeSlab = false;
        ma.close();
      } finally {
        System.clearProperty(MemoryAllocatorImpl.FREE_OFF_HEAP_MEMORY_PROPERTY);
      }
    } finally {
      if (freeSlab) {
        MemoryAllocatorImpl.freeOffHeapMemory();
      }
    }

  }

  @Test
  public void testDefragmentation() {
    final int perObjectOverhead = OffHeapStoredObject.HEADER_SIZE;
    final int BIG_ALLOC_SIZE = 150000;
    final int SMALL_ALLOC_SIZE = BIG_ALLOC_SIZE / 2;
    final int TOTAL_MEM = BIG_ALLOC_SIZE;
    SlabImpl slab = new SlabImpl(TOTAL_MEM);
    try {
      MemoryAllocatorImpl ma =
          MemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(),
              new NullOffHeapMemoryStats(), new SlabImpl[] {slab});
      StoredObject bmc = ma.allocate(BIG_ALLOC_SIZE - perObjectOverhead);
      try {
        StoredObject smc = ma.allocate(SMALL_ALLOC_SIZE - perObjectOverhead);
        fail("Expected out of memory");
      } catch (OutOfOffHeapMemoryException expected) {
      }
      bmc.release();
      assertEquals(TOTAL_MEM, ma.freeList.getFreeMemory());
      StoredObject smc1 = ma.allocate(SMALL_ALLOC_SIZE - perObjectOverhead);
      StoredObject smc2 = ma.allocate(SMALL_ALLOC_SIZE - perObjectOverhead);
      smc2.release();
      assertEquals(TOTAL_MEM - SMALL_ALLOC_SIZE, ma.freeList.getFreeMemory());
      try {
        bmc = ma.allocate(BIG_ALLOC_SIZE - perObjectOverhead);
        fail("Expected out of memory");
      } catch (OutOfOffHeapMemoryException expected) {
      }
      smc1.release();
      assertEquals(TOTAL_MEM, ma.freeList.getFreeMemory());
      bmc = ma.allocate(BIG_ALLOC_SIZE - perObjectOverhead);
      bmc.release();
      assertEquals(TOTAL_MEM, ma.freeList.getFreeMemory());
      ArrayList<StoredObject> mcs = new ArrayList<StoredObject>();
      for (int i = 0; i < BIG_ALLOC_SIZE / (8 + perObjectOverhead); i++) {
        mcs.add(ma.allocate(8));
      }
      checkMcs(mcs);
      assertEquals(0, ma.freeList.getFreeMemory());
      try {
        ma.allocate(8);
        fail("expected out of memory");
      } catch (OutOfOffHeapMemoryException expected) {
      }
      mcs.remove(0).release(); // frees 8+perObjectOverhead
      assertEquals(8 + perObjectOverhead, ma.freeList.getFreeMemory());
      mcs.remove(0).release(); // frees 8+perObjectOverhead
      assertEquals((8 + perObjectOverhead) * 2, ma.freeList.getFreeMemory());
      ma.allocate(16).release(); // allocates and frees 16+perObjectOverhead; still have
                                 // perObjectOverhead
      assertEquals((8 + perObjectOverhead) * 2, ma.freeList.getFreeMemory());
      mcs.remove(0).release(); // frees 8+perObjectOverhead
      assertEquals((8 + perObjectOverhead) * 3, ma.freeList.getFreeMemory());
      mcs.remove(0).release(); // frees 8+perObjectOverhead
      assertEquals((8 + perObjectOverhead) * 4, ma.freeList.getFreeMemory());
      // At this point I should have 8*4 + perObjectOverhead*4 of free memory
      ma.allocate(8 * 4 + perObjectOverhead * 3).release();
      assertEquals((8 + perObjectOverhead) * 4, ma.freeList.getFreeMemory());
      mcs.remove(0).release(); // frees 8+perObjectOverhead
      assertEquals((8 + perObjectOverhead) * 5, ma.freeList.getFreeMemory());
      // At this point I should have 8*5 + perObjectOverhead*5 of free memory
      try {
        ma.allocate((8 * 5 + perObjectOverhead * 4) + 1);
        fail("expected out of memory");
      } catch (OutOfOffHeapMemoryException expected) {
      }
      mcs.remove(0).release(); // frees 8+perObjectOverhead
      assertEquals((8 + perObjectOverhead) * 6, ma.freeList.getFreeMemory());
      checkMcs(mcs);
      // At this point I should have 8*6 + perObjectOverhead*6 of free memory
      StoredObject mc24 = ma.allocate(24);
      checkMcs(mcs);
      assertEquals((8 + perObjectOverhead) * 6 - (24 + perObjectOverhead),
          ma.freeList.getFreeMemory());
      // At this point I should have 8*3 + perObjectOverhead*5 of free memory
      StoredObject mc16 = ma.allocate(16);
      checkMcs(mcs);
      assertEquals(
          (8 + perObjectOverhead) * 6 - (24 + perObjectOverhead) - (16 + perObjectOverhead),
          ma.freeList.getFreeMemory());
      // At this point I should have 8*1 + perObjectOverhead*4 of free memory
      mcs.add(ma.allocate(8));
      checkMcs(mcs);
      assertEquals((8 + perObjectOverhead) * 6 - (24 + perObjectOverhead) - (16 + perObjectOverhead)
          - (8 + perObjectOverhead), ma.freeList.getFreeMemory());
      // At this point I should have 8*0 + perObjectOverhead*3 of free memory
      StoredObject mcDO = ma.allocate(perObjectOverhead * 2);
      checkMcs(mcs);
      // At this point I should have 8*0 + perObjectOverhead*0 of free memory
      assertEquals(0, ma.freeList.getFreeMemory());
      try {
        ma.allocate(1);
        fail("expected out of memory");
      } catch (OutOfOffHeapMemoryException expected) {
      }
      checkMcs(mcs);
      assertEquals(0, ma.freeList.getFreeMemory());
      mcDO.release();
      assertEquals((perObjectOverhead * 3), ma.freeList.getFreeMemory());
      mcs.remove(mcs.size() - 1).release();
      assertEquals((perObjectOverhead * 3) + (8 + perObjectOverhead), ma.freeList.getFreeMemory());
      mc16.release();
      assertEquals((perObjectOverhead * 3) + (8 + perObjectOverhead) + (16 + perObjectOverhead),
          ma.freeList.getFreeMemory());
      mc24.release();
      assertEquals((perObjectOverhead * 3) + (8 + perObjectOverhead) + (16 + perObjectOverhead)
          + (24 + perObjectOverhead), ma.freeList.getFreeMemory());

      long freeMem = ma.freeList.getFreeMemory();
      for (StoredObject mc : mcs) {
        mc.release();
        assertEquals(freeMem + (8 + perObjectOverhead), ma.freeList.getFreeMemory());
        freeMem += (8 + perObjectOverhead);
      }
      mcs.clear();
      assertEquals(TOTAL_MEM, ma.freeList.getFreeMemory());
      bmc = ma.allocate(BIG_ALLOC_SIZE - perObjectOverhead);
      bmc.release();
    } finally {
      MemoryAllocatorImpl.freeOffHeapMemory();
    }
  }

  @Test
  public void testUsageEventListener() {
    final int perObjectOverhead = OffHeapStoredObject.HEADER_SIZE;
    final int SMALL_ALLOC_SIZE = 1000;
    SlabImpl slab = new SlabImpl(3000);
    try {
      MemoryAllocatorImpl ma =
          MemoryAllocatorImpl.createForUnitTest(new NullOutOfOffHeapMemoryListener(),
              new NullOffHeapMemoryStats(), new SlabImpl[] {slab});
      MemoryUsageListener listener = new MemoryUsageListener() {
        @Override
        public void updateMemoryUsed(final long bytesUsed) {
          MemoryAllocatorJUnitTest.this.memoryUsageEventReceived = true;
          assertEquals(MemoryAllocatorJUnitTest.this.expectedMemoryUsage, bytesUsed);
        }
      };
      ma.addMemoryUsageListener(listener);

      this.expectedMemoryUsage = SMALL_ALLOC_SIZE;
      this.memoryUsageEventReceived = false;
      StoredObject smc = ma.allocate(SMALL_ALLOC_SIZE - perObjectOverhead);
      assertEquals(true, this.memoryUsageEventReceived);

      this.expectedMemoryUsage = SMALL_ALLOC_SIZE * 2;
      this.memoryUsageEventReceived = false;
      smc = ma.allocate(SMALL_ALLOC_SIZE - perObjectOverhead);
      assertEquals(true, this.memoryUsageEventReceived);

      MemoryUsageListener unaddedListener = new MemoryUsageListener() {
        @Override
        public void updateMemoryUsed(final long bytesUsed) {
          throw new IllegalStateException("Should never be called");
        }
      };
      ma.removeMemoryUsageListener(unaddedListener);

      ma.removeMemoryUsageListener(listener);

      ma.removeMemoryUsageListener(unaddedListener);

      this.expectedMemoryUsage = SMALL_ALLOC_SIZE * 2;
      this.memoryUsageEventReceived = false;
      smc = ma.allocate(SMALL_ALLOC_SIZE - perObjectOverhead);
      assertEquals(false, this.memoryUsageEventReceived);

    } finally {
      MemoryAllocatorImpl.freeOffHeapMemory();
    }
  }

  private void checkMcs(ArrayList<StoredObject> mcs) {
    for (StoredObject mc : mcs) {
      assertEquals(8 + 8, mc.getSize());
    }
  }

  @Test
  public void testOutOfOffHeapMemory() {
    final int perObjectOverhead = OffHeapStoredObject.HEADER_SIZE;
    final int BIG_ALLOC_SIZE = 150000;
    final int SMALL_ALLOC_SIZE = BIG_ALLOC_SIZE / 2;
    final int TOTAL_MEM = BIG_ALLOC_SIZE;
    final SlabImpl slab = new SlabImpl(TOTAL_MEM);
    final AtomicReference<OutOfOffHeapMemoryException> ooom =
        new AtomicReference<OutOfOffHeapMemoryException>();
    final OutOfOffHeapMemoryListener oooml = new OutOfOffHeapMemoryListener() {
      @Override
      public void outOfOffHeapMemory(OutOfOffHeapMemoryException cause) {
        ooom.set(cause);
      }

      @Override
      public void close() {}
    };
    try {
      MemoryAllocatorImpl ma = MemoryAllocatorImpl.createForUnitTest(oooml,
          new NullOffHeapMemoryStats(), new SlabImpl[] {slab});
      // make a big allocation
      StoredObject bmc = ma.allocate(BIG_ALLOC_SIZE - perObjectOverhead);
      assertNull(ooom.get());
      // drive the ma to ooom with small allocations
      try {
        StoredObject smc = ma.allocate(SMALL_ALLOC_SIZE - perObjectOverhead);
        fail("Expected out of memory");
      } catch (OutOfOffHeapMemoryException expected) {
      }
      assertNotNull(ooom.get());
      assertTrue(
          ooom.get().getMessage().contains("Out of off-heap memory. Could not allocate size of "));
    } finally {
      MemoryAllocatorImpl.freeOffHeapMemory();
    }
  }
}


text:
package org.apache.coyote.http11;

import org.apache.tomcat.util.buf.ByteChunk;

/**
 * Constants.
 *
 * @author Remy Maucherat
 */
public final class Constants {

    public static final int DEFAULT_CONNECTION_TIMEOUT = 60000;

    /**
     * CRLF.
     */
    public static final String CRLF = "\r\n";

    /**
     * CR.
     */
    public static final byte CR = (byte) '\r';

    /**
     * LF.
     */
    public static final byte LF = (byte) '\n';

    /**
     * SP.
     */
    public static final byte SP = (byte) ' ';

    /**
     * HT.
     */
    public static final byte HT = (byte) '\t';

    /**
     * COLON.
     */
    public static final byte COLON = (byte) ':';

    /**
     * SEMI_COLON.
     */
    public static final byte SEMI_COLON = (byte) ';';

    /**
     * 'A'.
     */
    public static final byte A = (byte) 'A';

    /**
     * 'a'.
     */
    public static final byte a = (byte) 'a';

    /**
     * 'Z'.
     */
    public static final byte Z = (byte) 'Z';

    /**
     * '?'.
     */
    public static final byte QUESTION = (byte) '?';

    /**
     * Lower case offset.
     */
    public static final byte LC_OFFSET = A - a;

    /* Various constant "strings" */
    public static final String CONNECTION = "Connection";
    public static final String CLOSE = "close";
    public static final String KEEP_ALIVE_HEADER_VALUE_TOKEN = "keep-alive";
    public static final String CHUNKED = "chunked";
    public static final byte[] ACK_BYTES = ByteChunk.convertToBytes("HTTP/1.1 100 " + CRLF + CRLF);
    public static final String TRANSFERENCODING = "Transfer-Encoding";
    public static final String KEEP_ALIVE_HEADER_NAME = "Keep-Alive";
    public static final byte[] _200_BYTES = ByteChunk.convertToBytes("200");
    public static final byte[] _400_BYTES = ByteChunk.convertToBytes("400");
    public static final byte[] _404_BYTES = ByteChunk.convertToBytes("404");

    /**
     * Identity filters (input and output).
     */
    public static final int IDENTITY_FILTER = 0;

    /**
     * Chunked filters (input and output).
     */
    public static final int CHUNKED_FILTER = 1;

    /**
     * Void filters (input and output).
     */
    public static final int VOID_FILTER = 2;

    /**
     * GZIP filter (output).
     */
    public static final int GZIP_FILTER = 3;

    /**
     * Buffered filter (input)
     */
    public static final int BUFFERED_FILTER = 3;

    /**
     * HTTP/1.0.
     */
    public static final String HTTP_10 = "HTTP/1.0";

    /**
     * HTTP/1.1.
     */
    public static final String HTTP_11 = "HTTP/1.1";
    public static final byte[] HTTP_11_BYTES = ByteChunk.convertToBytes(HTTP_11);
}

redpajama_set_name: RedPajamaGithub


text:
package com.amazonaws.services.ec2.model;

import java.io.Serializable;
import javax.annotation.Generated;

/**
 *

 * Describes a managed prefix list.
 *

 *
 * @see AWS API
 *      Documentation
 */
@Generated("com.amazonaws:aws-java-sdk-code-generator")
public class ManagedPrefixList implements Serializable, Cloneable {

    /**
     *

     * The ID of the prefix list.
     *

     */
    private String prefixListId;
    /**
     *

     * The IP address version.
     *

     */
    private String addressFamily;
    /**
     *

     * The current state of the prefix list.
     *

     */
    private String state;
    /**
     *

     * The state message.
     *

     */
    private String stateMessage;
    /**
     *

     * The Amazon Resource Name (ARN) for the prefix list.
     *

     */
    private String prefixListArn;
    /**
     *

     * The name of the prefix list.
     *

     */
    private String prefixListName;
    /**
     *

     * The maximum number of entries for the prefix list.
     *

     */
    private Integer maxEntries;
    /**
     *

     * The version of the prefix list.
     *

     */
    private Long version;
    /**
     *

     * The tags for the prefix list.
     *

     */
    private com.amazonaws.internal.SdkInternalList<Tag> tags;
    /**
     *

     * The ID of the owner of the prefix list.
     *

     */
    private String ownerId;

    /**
     *

     * The ID of the prefix list.
     *

     *
     * @param prefixListId
     *        The ID of the prefix list.
     */

    public void setPrefixListId(String prefixListId) {
        this.prefixListId = prefixListId;
    }

    /**
     *

     * The ID of the prefix list.
     *

     *
     * @return The ID of the prefix list.
     */

    public String getPrefixListId() {
        return this.prefixListId;
    }

    /**
     *

     * The ID of the prefix list.
     *

     *
     * @param prefixListId
     *        The ID of the prefix list.
     * @return Returns a reference to this object so that method calls can be chained together.
     */

    public ManagedPrefixList withPrefixListId(String prefixListId) {
        setPrefixListId(prefixListId);
        return this;
    }

    /**
     *

     * The IP address version.
     *

     *
     * @param addressFamily
     *        The IP address version.
     */

    public void setAddressFamily(String addressFamily) {
        this.addressFamily = addressFamily;
    }

    /**
     *

     * The IP address version.
     *

     *
     * @return The IP address version.
     */

    public String getAddressFamily() {
        return this.addressFamily;
    }

    /**
     *

     * The IP address version.
     *

     *
     * @param addressFamily
     *        The IP address version.
     * @return Returns a reference to this object so that method calls can be chained together.
     */

    public ManagedPrefixList withAddressFamily(String addressFamily) {
        setAddressFamily(addressFamily);
        return this;
    }

    /**
     *

     * The current state of the prefix list.
     *

     *
     * @param state
     *        The current state of the prefix list.
     * @see PrefixListState
     */

    public void setState(String state) {
        this.state = state;
    }

    /**
     *

     * The current state of the prefix list.
     *

     *
     * @return The current state of the prefix list.
     * @see PrefixListState
     */

    public String getState() {
        return this.state;
    }

    /**
     *

     * The current state of the prefix list.
     *

     *
     * @param state
     *        The current state of the prefix list.
     * @return Returns a reference to this object so that method calls can be chained together.
     * @see PrefixListState
     */

    public ManagedPrefixList withState(String state) {
        setState(state);
        return this;
    }

    /**
     *

     * The current state of the prefix list.
     *

     *
     * @param state
     *        The current state of the prefix list.
     * @return Returns a reference to this object so that method calls can be chained together.
     * @see PrefixListState
     */

    public ManagedPrefixList withState(PrefixListState state) {
        this.state = state.toString();
        return this;
    }

    /**
     *

     * The state message.
     *

     *
     * @param stateMessage
     *        The state message.
     */

    public void setStateMessage(String stateMessage) {
        this.stateMessage = stateMessage;
    }

    /**
     *

     * The state message.
     *

     *
     * @return The state message.
     */

    public String getStateMessage() {
        return this.stateMessage;
    }

    /**
     *

     * The state message.
     *

     *
     * @param stateMessage
     *        The state message.
     * @return Returns a reference to this object so that method calls can be chained together.
     */

    public ManagedPrefixList withStateMessage(String stateMessage) {
        setStateMessage(stateMessage);
        return this;
    }

    /**
     *

     * The Amazon Resource Name (ARN) for the prefix list.
     *

     *
     * @param prefixListArn
     *        The Amazon Resource Name (ARN) for the prefix list.
     */

    public void setPrefixListArn(String prefixListArn) {
        this.prefixListArn = prefixListArn;
    }

    /**
     *

     * The Amazon Resource Name (ARN) for the prefix list.
     *

     *
     * @return The Amazon Resource Name (ARN) for the prefix list.
     */

    public String getPrefixListArn() {
        return this.prefixListArn;
    }

    /**
     *

     * The Amazon Resource Name (ARN) for the prefix list.
     *

     *
     * @param prefixListArn
     *        The Amazon Resource Name (ARN) for the prefix list.
     * @return Returns a reference to this object so that method calls can be chained together.
     */

    public ManagedPrefixList withPrefixListArn(String prefixListArn) {
        setPrefixListArn(prefixListArn);
        return this;
    }

    /**
     *

     * The name of the prefix list.
     *

     *
     * @param prefixListName
     *        The name of the prefix list.
     */

    public void setPrefixListName(String prefixListName) {
        this.prefixListName = prefixListName;
    }

    /**
     *

     * The name of the prefix list.
     *

     *
     * @return The name of the prefix list.
     */

    public String getPrefixListName() {
        return this.prefixListName;
    }

    /**
     *

     * The name of the prefix list.
     *

     *
     * @param prefixListName
     *        The name of the prefix list.
     * @return Returns a reference to this object so that method calls can be chained together.
     */

    public ManagedPrefixList withPrefixListName(String prefixListName) {
        setPrefixListName(prefixListName);
        return this;
    }

    /**
     *

     * The maximum number of entries for the prefix list.
     *

     *
     * @param maxEntries
     *        The maximum number of entries for the prefix list.
     */

    public void setMaxEntries(Integer maxEntries) {
        this.maxEntries = maxEntries;
    }

    /**
     *

     * The maximum number of entries for the prefix list.
     *

     *
     * @return The maximum number of entries for the prefix list.
     */

    public Integer getMaxEntries() {
        return this.maxEntries;
    }

    /**
     *

     * The maximum number of entries for the prefix list.
     *

     *
     * @param maxEntries
     *        The maximum number of entries for the prefix list.
     * @return Returns a reference to this object so that method calls can be chained together.
     */

    public ManagedPrefixList withMaxEntries(Integer maxEntries) {
        setMaxEntries(maxEntries);
        return this;
    }

    /**
     *

     * The version of the prefix list.
     *

     *
     * @param version
     *        The version of the prefix list.
     */

    public void setVersion(Long version) {
        this.version = version;
    }

    /**
     *

     * The version of the prefix list.
     *

     *
     * @return The version of the prefix list.
     */

    public Long getVersion() {
        return this.version;
    }

    /**
     *

     * The version of the prefix list.
     *

     *
     * @param version
     *        The version of the prefix list.
     * @return Returns a reference to this object so that method calls can be chained together.
     */

    public ManagedPrefixList withVersion(Long version) {
        setVersion(version);
        return this;
    }

    /**
     *

     * The tags for the prefix list.
     *

     *
     * @return The tags for the prefix list.
     */

    public java.util.List<Tag> getTags() {
        if (tags == null) {
            tags = new com.amazonaws.internal.SdkInternalList<Tag>();
        }
        return tags;
    }

    /**
     *

     * The tags for the prefix list.
     *

     *
     * @param tags
     *        The tags for the prefix list.
     */

    public void setTags(java.util.Collection<Tag> tags) {
        if (tags == null) {
            this.tags = null;
            return;
        }

        this.tags = new com.amazonaws.internal.SdkInternalList<Tag>(tags);
    }

    /**
     *

     * The tags for the prefix list.
     *

     *

     * NOTE: This method appends the values to the existing list (if any). Use
     * {@link #setTags(java.util.Collection)} or {@link #withTags(java.util.Collection)} if you want to override the
     * existing values.
     *

     *
     * @param tags
     *        The tags for the prefix list.
     * @return Returns a reference to this object so that method calls can be chained together.
     */

    public ManagedPrefixList withTags(Tag... tags) {
        if (this.tags == null) {
            setTags(new com.amazonaws.internal.SdkInternalList<Tag>(tags.length));
        }
        for (Tag ele : tags) {
            this.tags.add(ele);
        }
        return this;
    }

    /**
     *

     * The tags for the prefix list.
     *

     *
     * @param tags
     *        The tags for the prefix list.
     * @return Returns a reference to this object so that method calls can be chained together.
     */

    public ManagedPrefixList withTags(java.util.Collection<Tag> tags) {
        setTags(tags);
        return this;
    }

    /**
     *

     * The ID of the owner of the prefix list.
     *

     *
     * @param ownerId
     *        The ID of the owner of the prefix list.
     */

    public void setOwnerId(String ownerId) {
        this.ownerId = ownerId;
    }

    /**
     *

     * The ID of the owner of the prefix list.
     *

     *
     * @return The ID of the owner of the prefix list.
     */

    public String getOwnerId() {
        return this.ownerId;
    }

    /**
     *

     * The ID of the owner of the prefix list.
     *

     *
     * @param ownerId
     *        The ID of the owner of the prefix list.
     * @return Returns a reference to this object so that method calls can be chained together.
     */

    public ManagedPrefixList withOwnerId(String ownerId) {
        setOwnerId(ownerId);
        return this;
    }

    /**
     * Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be
     * redacted from this string using a placeholder value.
     *
     * @return A string representation of this object.
     *
     * @see java.lang.Object#toString()
     */
    @Override
    public String toString() {
        StringBuilder sb = new StringBuilder();
        sb.append("{");
        if (getPrefixListId() != null)
            sb.append("PrefixListId: ").append(getPrefixListId()).append(",");
        if (getAddressFamily() != null)
            sb.append("AddressFamily: ").append(getAddressFamily()).append(",");
        if (getState() != null)
            sb.append("State: ").append(getState()).append(",");
        if (getStateMessage() != null)
            sb.append("StateMessage: ").append(getStateMessage()).append(",");
        if (getPrefixListArn() != null)
            sb.append("PrefixListArn: ").append(getPrefixListArn()).append(",");
        if (getPrefixListName() != null)
            sb.append("PrefixListName: ").append(getPrefixListName()).append(",");
        if (getMaxEntries() != null)
            sb.append("MaxEntries: ").append(getMaxEntries()).append(",");
        if (getVersion() != null)
            sb.append("Version: ").append(getVersion()).append(",");
        if (getTags() != null)
            sb.append("Tags: ").append(getTags()).append(",");
        if (getOwnerId() != null)
            sb.append("OwnerId: ").append(getOwnerId());
        sb.append("}");
        return sb.toString();
    }

    @Override
    public boolean equals(Object obj) {
        if (this == obj)
            return true;
        if (obj == null)
            return false;

        if (obj instanceof ManagedPrefixList == false)
            return false;
        ManagedPrefixList other = (ManagedPrefixList) obj;
        if (other.getPrefixListId() == null ^ this.getPrefixListId() == null)
            return false;
        if (other.getPrefixListId() != null && other.getPrefixListId().equals(this.getPrefixListId()) == false)
            return false;
        if (other.getAddressFamily() == null ^ this.getAddressFamily() == null)
            return false;
        if (other.getAddressFamily() != null && other.getAddressFamily().equals(this.getAddressFamily()) == false)
            return false;
        if (other.getState() == null ^ this.getState() == null)
            return false;
        if (other.getState() != null && other.getState().equals(this.getState()) == false)
            return false;
        if (other.getStateMessage() == null ^ this.getStateMessage() == null)
            return false;
        if (other.getStateMessage() != null && other.getStateMessage().equals(this.getStateMessage()) == false)
            return false;
        if (other.getPrefixListArn() == null ^ this.getPrefixListArn() == null)
            return false;
        if (other.getPrefixListArn() != null && other.getPrefixListArn().equals(this.getPrefixListArn()) == false)
            return false;
        if (other.getPrefixListName() == null ^ this.getPrefixListName() == null)
            return false;
        if (other.getPrefixListName() != null && other.getPrefixListName().equals(this.getPrefixListName()) == false)
            return false;
        if (other.getMaxEntries() == null ^ this.getMaxEntries() == null)
            return false;
        if (other.getMaxEntries() != null && other.getMaxEntries().equals(this.getMaxEntries()) == false)
            return false;
        if (other.getVersion() == null ^ this.getVersion() == null)
            return false;
        if (other.getVersion() != null && other.getVersion().equals(this.getVersion()) == false)
            return false;
        if (other.getTags() == null ^ this.getTags() == null)
            return false;
        if (other.getTags() != null && other.getTags().equals(this.getTags()) == false)
            return false;
        if (other.getOwnerId() == null ^ this.getOwnerId() == null)
            return false;
        if (other.getOwnerId() != null && other.getOwnerId().equals(this.getOwnerId()) == false)
            return false;
        return true;
    }

    @Override
    public int hashCode() {
        final int prime = 31;
        int hashCode = 1;

        hashCode = prime * hashCode + ((getPrefixListId() == null) ? 0 : getPrefixListId().hashCode());
        hashCode = prime * hashCode + ((getAddressFamily() == null) ? 0 : getAddressFamily().hashCode());
        hashCode = prime * hashCode + ((getState() == null) ? 0 : getState().hashCode());
        hashCode = prime * hashCode + ((getStateMessage() == null) ? 0 : getStateMessage().hashCode());
        hashCode = prime * hashCode + ((getPrefixListArn() == null) ? 0 : getPrefixListArn().hashCode());
        hashCode = prime * hashCode + ((getPrefixListName() == null) ? 0 : getPrefixListName().hashCode());
        hashCode = prime * hashCode + ((getMaxEntries() == null) ? 0 : getMaxEntries().hashCode());
        hashCode = prime * hashCode + ((getVersion() == null) ? 0 : getVersion().hashCode());
        hashCode = prime * hashCode + ((getTags() == null) ? 0 : getTags().hashCode());
        hashCode = prime * hashCode + ((getOwnerId() == null) ? 0 : getOwnerId().hashCode());
        return hashCode;
    }

    @Override
    public ManagedPrefixList clone() {
        try {
            return (ManagedPrefixList) super.clone();
        } catch (CloneNotSupportedException e) {
            throw new IllegalStateException("Got a CloneNotSupportedException from Object.clone() " + "even though we're Cloneable!", e);
        }
    }

}


text:
Use DMadmin
Exec dbo.DDBMABackup @ClientHost = 'SQL2012.ddlab.local',
	@BackupLevel = 'full',
	@BackupSetName = 'ContactsFull',
	@BackupSetDescription = 'Contacts Full Backup',
	@RetentionDays = 7,
	@DDHost = 'dd640-01.ddlab.local',
	@DDBoostUser = 'ddboost',
	@DDStorageUnit = '/Boost-SQL',
	@SQLDatabaseName = 'Contacts'

-- Incr/Log Backup Production to primary DD - Run from primary
Use DMadmin
Exec dbo.DDBMABackup @ClientHost = 'SQL2012.ddlab.local',
	@BackupLevel = 'incr',
	@BackupSetName = 'ContactsLog',
	@BackupSetDescription = 'Contacts Transaction Log Backup',
	@RetentionDays = 7,
	@DDHost = 'dd640-01.ddlab.local',
	@DDBoostUser = 'ddboost',
	@DDStorageUnit = '/Boost-SQL',
	@SQLDatabaseName = 'Contacts'

-- Full backup of system databases (master,model,msdb)
declare @cur cursor;
declare @dbname nvarchar(100);
declare @bsetname nvarchar(255);
declare @bsetdescription nvarchar(255);
DECLARE @t TABLE (DBNAME NVARCHAR(100))
INSERT INTO @t
	Select Name From Sys.Databases Where (name <> 'tempdb') and (database_id < 5)
SET @cur = Cursor FOR
SELECT DBNAME FROM @t
open @cur
fetch next FROM @cur INTO @dbname
while @@fetch_status = 0
begin
	print '--------------------------------------------------------------'
	print 'Backing up ' + @dbname
	print '--------------------------------------------------------------'

	Set @bsetname = @dbname + '_Full';
	Set @bsetdescription = @dbname + ' Full Backup';

	Use DMadmin
	Exec dbo.DDBMABackup @ClientHost = 'SQL2012.ddlab.local',
		@BackupLevel = 'full',
		@BackupSetName = @bsetname,
		@BackupSetDescription = @bsetdescription,
		@RetentionDays = 7,
		@DDHost = 'dd640-01.ddlab.local',
		@DDBoostUser = 'ddboost',
		@DDStorageUnit = '/Boost-SQL',
		@SQLDatabaseName = @dbname

	fetch next FROM @cur INTO @dbname
end
close @cur;
Deallocate @cur;

-- Full backup of all user databases
declare @cur cursor;
declare @dbname nvarchar(100);
declare @bsetname nvarchar(255);
declare @bsetdescription nvarchar(255);
DECLARE @t TABLE (DBNAME NVARCHAR(100))
INSERT INTO @t
	Select Name From Sys.Databases Where (database_id > 5)
SET @cur = Cursor FOR
SELECT DBNAME FROM @t
open @cur
fetch next FROM @cur INTO @dbname
while @@fetch_status = 0
begin
	print '--------------------------------------------------------------'
	print 'Backing up ' + @dbname
	print '--------------------------------------------------------------'

	Set @bsetname = @dbname + '_Full';
	Set @bsetdescription = @dbname + ' Full Backup';

	Use DMadmin
	Exec dbo.DDBMABackup @ClientHost = 'SQL2012.ddlab.local',
		@BackupLevel = 'full',
		@BackupSetName = @bsetname,
		@BackupSetDescription = @bsetdescription,
		@RetentionDays = 7,
		@DDHost = 'dd640-01.ddlab.local',
		@DDBoostUser = 'ddboost',
		@DDStorageUnit = '/Boost-SQL',
		@SQLDatabaseName = @dbname

	fetch next FROM @cur INTO @dbname
end
close @cur;
Deallocate @cur;

-- Full backup of specific databases
declare @cur cursor;
declare @dbname nvarchar(100);
declare @bsetname nvarchar(255);
declare @bsetdescription nvarchar(255);
DECLARE @t TABLE (DBNAME NVARCHAR(100))
INSERT INTO @t
	Select Name From Sys.Databases Where Name in ('master','contacts')
SET @cur = Cursor FOR
SELECT DBNAME FROM @t
open @cur
fetch next FROM @cur INTO @dbname
while @@fetch_status = 0
begin
	print '--------------------------------------------------------------'
	print 'Backing up ' + @dbname
	print '--------------------------------------------------------------'

	Set @bsetname = @dbname + '_Full';
	Set @bsetdescription = @dbname + ' Full Backup';

	Use DMadmin
	Exec dbo.DDBMABackup @ClientHost = 'SQL2012.ddlab.local',
		@BackupLevel = 'full',
		@BackupSetName = @bsetname,
		@BackupSetDescription = @bsetdescription,
		@RetentionDays = 7,
		@DDHost = 'dd640-01.ddlab.local',
		@DDBoostUser = 'ddboost',
		@DDStorageUnit = '/Boost-SQL',
		@SQLDatabaseName = @dbname

	fetch next FROM @cur INTO @dbname
end
close @cur;
Deallocate @cur;

-- Restore databases - Note: to restore to different target run restore command on target server but specify source server in @ClientHost parameter
-- Restore backup from primary and complete recovery
Use DMadmin
Exec dbo.DDBMARestore @ClientHost = 'SQL2012.ddlab.local',
	@RecoveryState = 'normal',
	@DDHost = 'dd640-01.ddlab.local',
	@DDBoostUser = 'ddboost',
	@DDStorageUnit = '/Boost-SQL',
	@SQLDatabaseName = 'Contacts',
	@Overwrite = 1

-- Restore backup from secondary and complete recovery
Use DMadmin
Exec dbo.DDBMARestore @ClientHost = 'SQL2012.ddlab.local',
	@RecoveryState = 'normal',
	@DDHost = 'dd610-02.ddlab.local',
	@DDBoostUser = 'boost',
	@DDStorageUnit = '/Boost-SQL',
	@SQLDatabaseName = 'Contacts',
	@Overwrite = 1

-- Restore backup from secondary leave in recovery mode for transaction log restores
Use DMadmin
Exec dbo.DDBMARestore @ClientHost = 'SQL2012.ddlab.local',
	@BackupTimeStamp = '11/13/2014 22:32:25',
	@RecoveryState = 'norecover',
	@DDHost = 'dd640-01.ddlab.local',
	@DDBoostUser = 'ddboost',
	@DDStorageUnit = '/Boost-SQL',
	@SQLDatabaseName = 'Contacts',
	@Overwrite = 1

-- Complete recovery and leave database in normal state
Use DMadmin
Exec DDBMARestore @ClientHost = 'SQL2012.ddlab.local',
	@BackupTimeStamp = '11/14/2014 07:48:20',
	@RecoveryState = 'normal',
	@DDHost = 'dd610-02.ddlab.local',
	@DDBoostUser = 'boost',
	@DDStorageUnit = '/Boost-SQL',
	@SQLDatabaseName = 'Contacts'

-- Manager SQL backup catalog table
-- Get SQL Backup Catalog
Use DMadmin
Exec DDBMACatalog

-- Get SQL Backup Catalog entries for BackupSetName
Use DMadmin
Exec DDBMACatalog @BackupSetName = 'ContactsFull'

-- Get SQL Backup Catalog entries for DatabaseName
Use DMadmin
Exec dbo.DDBMACatalog @SQLDatabaseName = 'Contacts'

-- Get SQL Backup Catalog entries for time period based on backup date
Use DMadmin
Exec DDBMACatalog @BackupTimeStart = '12/25/2014', @BackupTimeEnd = '12/31/2014'

-- Get SQL Backup Catalog entries for specific database and time period based on expiration date
Use DMadmin
Exec DDBMACatalog @SQLDatabaseName = 'Contacts', @BackupExpireStart = '12/25/2014', @BackupExpireEnd = '12/31/2014'

-- Delete SQL Backup Catalog entries for time period based on expiration date
Use DMadmin
Exec DDBMACatalog

-- Delete Expired Entries between 90 and 100 days old
declare @StartDate datetime = dateadd(day,-50,getdate())
declare @EndDate datetime = dateadd(day,-1,getdate())
Exec DDBMACatalog @BackupExpireStart = @StartDate, @BackupExpireEnd = @EndDate

-- Manage expired backup images on Data Domain storage unit
-- Show all backup images
Use DMadmin
Exec dbo.DDBMAExpire @AppType = 'mssql',
	@ConfigFile = 'C:\dd640-dm.cfg'

-- Show specific backup images
Use DMadmin
Exec dbo.DDBMAExpire @AppType = 'mssql',
	@ConfigFile = 'C:\dd640-dm.cfg',
	@StartTime = '11/18/2014 10:45:40',
	@EndTime = '11/18/2014 10:45:40',
	@SaveSet = 'Contacts_Full',
	@DelExpired = NULL,
	@Clean = NULL,
	@Verbose = NULL

-- Delete specific backup images
Use DMadmin
Exec dbo.DDBMAExpire @AppType = 'mssql',
	@ConfigFile = 'C:\dd640-dm.cfg',
	@StartTime = '11/18/2014 10:45:40',
	@EndTime = '11/18/2014 10:45:40',
	@SaveSet = 'Contacts_Full',
	@DelExpired = 1,
	@Clean = NULL,
	@Verbose = NULL

-- Delete Expired Entries between 90 and 100 days old
Use DMadmin
declare @StartDatetime varchar(25) = convert(varchar(25),Format(dateadd(day,-100,getdate()), 'MM/dd/yyyy hh:mm:ss'))
declare @EndDatetime varchar(25) = convert(varchar(25),Format(dateadd(day,-90,getdate()), 'MM/dd/yyyy hh:mm:ss'))

Exec dbo.DDBMAExpire @AppType = 'mssql',
	@ConfigFile = 'C:\dd640-dm.cfg',
	@StartTime = '11/21/2014 00:00:01',
	@EndTime = '11/21/2014 23:59:59',
	@SaveSet = 'ContactsFull',
	@DelExpired = 1,
	@Clean = NULL,
	@Verbose = NULL


text:
<?php
/* import all twig files */
Application::Import('Twig::Autoloader.php');
Twig_Autoloader::register();

class TwigTemplate implements ITemplate {
		private $paths = array();

		public function addPath($path) {
			$this->paths[] = $path;
		}

		public function Render($view, $vars=array(), $custompath=null) {

			if($custompath == null) {
				/* build up view path */
				$path = Application::Path('views');

			}

			else {
				$path = $custompath;
			}

			$view = str_replace("::", "/", $view);

			if(file_exists($path . $view)) {
				$loader = new Twig_Loader_Filesystem($path);
			}

			else {
				/* lookup paths to find view */
				foreach($this->paths as $path) {

					if(file_exists($path . $view)) {
						$loader = new Twig_Loader_Filesystem($path);
					}
				}
			}

			$twig = new Twig_Environment($loader, array(
			    'cache' => Application::Path('temp'). 'cache/templates',
			    'auto_reload' => true,
			    'debug' => true,
			));

			$twig->addGlobal('app', Application::instance());
			$twig->addGlobal('lang', Language::instance());
			$twig->addGlobal('request', HTTPRequest::instance());
			$twig->addGlobal('response', HTTPResponse::instance());
			$twig->addGlobal('session', Session::instance());
			$twig->addGlobal('auth', Auth::instance());

			/* include all elements */
			$elements = glob(Application::path('elements') . '*' , GLOB_ONLYDIR);

			foreach($elements as $element) {
				/* setup default object */

				$element_raw = basename($element);
				$element_parts = explode('@', $element_raw);
				$element_name = $element_parts[0]. 'Element';

				$twig->addGlobal($element_parts[0], new $element_name());

			}

			$twig->addExtension(new Twig_Extension_Debug());

			echo $twig->render($view, $vars);

		}

		public function Buffer($view, $vars=array(), $custompath=null) {

			if($custompath == null) {
				/* build up view path */
				$path = Application::Path('views');

			}

			else {
				$path = $custompath;
			}

			$view = str_replace("::", "/", $view);

			if(file_exists($path . $view)) {
				$loader = new Twig_Loader_Filesystem($path);
			}

			else {
				/* lookup paths to find view */
				foreach($this->paths as $path) {
					if(file_exists($path . $view)) {
						$loader = new Twig_Loader_Filesystem($path);
					}
				}
			}

			$twig = new Twig_Environment($loader, array(
			    'cache' => Application::Path('temp'). 'cache/templates',
			    'debug' => true,
			));

			$twig->addGlobal('app', Application::instance());
			$twig->addGlobal('lang', Language::instance());
			$twig->addGlobal('request', HTTPRequest::instance());
			$twig->addGlobal('response', HTTPResponse::instance());
			$twig->addGlobal('session', Session::instance());
			$twig->addGlobal('auth', Auth::instance());

			/* include all elements */
			$elements = glob(Application::path('elements') . '*' , GLOB_ONLYDIR);

			foreach($elements as $element) {
				/* setup default object */

				$element_raw = basename($element);
				$element_parts = explode('@', $element_raw);
				$element_name = $element_parts[0]. 'Element';

				$twig->addGlobal($element_parts[0], new $element_name());

			}

			$twig->addExtension(new Twig_Extension_Debug());

				return $twig->render($view, $vars);

		}
	}
?>


text:
import praw
import inspect, os, sys # set the BASE_DIR
import simplejson as json
import datetime, yaml, time, csv, pytz
import reddit.connection
import reddit.praw_utils as praw_utils
import reddit.queries
import sqlalchemy
from collections import defaultdict
from dateutil import parser
from utils.common import *
from app.models import Base, SubredditPage, Subreddit, Post, ModAction, PrawKey, Comment
from app.models import Experiment, ExperimentThing, ExperimentAction, ExperimentThingSnapshot
from app.models import EventHook
from sqlalchemy import and_, or_, desc, asc
from app.controllers.subreddit_controller import SubredditPageController

### LOAD ENVIRONMENT VARIABLES
BASE_DIR = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "..","..")
ENV = os.environ['CS_ENV']

class StylesheetExperimentController:
    def __init__(self, experiment_name, db_session, r, log,
                       required_keys = ['subreddit', 'subreddit_id', 'username',
                         'start_time', 'end_time', 'conditions',
                         'intervention_interval_seconds', 'intervention_window_seconds',
                         'first_n_comments','comment_snapshot_period_seconds']):
        self.db_session = db_session
        self.log = log
        self.r = r
        self.load_experiment_config(required_keys, experiment_name)

    def get_experiment_config(self, required_keys, experiment_name):
        experiment_file_path = os.path.join(BASE_DIR, "config", "experiments", experiment_name) + ".yml"
        with open(experiment_file_path, 'r') as f:
            try:
                experiment_config_all = yaml.full_load(f)
            except yaml.YAMLError as exc:
                self.log.error("{0}: Failure loading experiment yaml {1}".format(
                    self.__class__.__name__, experiment_file_path), str(exc))
                sys.exit(1)
        if(ENV not in experiment_config_all.keys()):
            self.log.error("{0}: Cannot find experiment settings for {1} in {2}".format(
                self.__class__.__name__, ENV, experiment_file_path))
            sys.exit(1)

        experiment_config = experiment_config_all[ENV]
        for key in required_keys:
            if key not in experiment_config.keys():
                self.log.error("{0}: Value missing from {1}: {2}".format(
                    self.__class__.__name__, experiment_file_path, key))
                sys.exit(1)
        return experiment_config

    def load_experiment_config(self, required_keys, experiment_name):
        experiment_config = self.get_experiment_config(required_keys, experiment_name)
        experiment = self.db_session.query(Experiment).filter(Experiment.name == experiment_name).first()
        if(experiment is None):

            condition_keys = []

            ## LOAD RANDOMIZED CONDITIONS (see CivilServant-Analysis)
            for condition in experiment_config['conditions'].values():
                with open(os.path.join(BASE_DIR, "config", "experiments", condition['randomizations']), "r") as f:
                    reader = csv.DictReader(f)
                    randomizations = []
                    for row in reader:
                        randomizations.append(row)
                        condition['randomizations']  = randomizations

            experiment = Experiment(
                name = experiment_name,
                controller = self.__class__.__name__,
                start_time = parser.parse(experiment_config['start_time']),
                end_time = parser.parse(experiment_config['end_time']),
                settings_json = json.dumps(experiment_config)
            )
            self.db_session.add(experiment)
            self.db_session.commit()

        ### SET UP INSTANCE PROPERTIES
        self.experiment = experiment
        self.experiment_settings = json.loads(self.experiment.settings_json)

        self.experiment_name = experiment_name

        self.subreddit = experiment_config['subreddit']
        self.subreddit_id = experiment_config['subreddit_id']
        self.username = experiment_config['username']

    def update_experiment(self):
        ## DETERMINE ELIGIBILITY
        if(self.determine_intervention_eligible()):
            condname = self.select_condition()
            self.run_intervention(condname)

    def determine_intervention_eligible(self):
        current_time = datetime.datetime.utcnow().replace(tzinfo=pytz.utc)
        start_time = parser.parse(self.experiment_settings['start_time'])
        end_time = parser.parse(self.experiment_settings['end_time'])
        eligible = ((current_time > start_time) and (current_time < end_time))

        last_experiment_created = None

        if(eligible):
            last_experiment_action = self.db_session.query(ExperimentAction).filter(
                ExperimentAction.experiment_id==self.experiment.id).order_by(desc(ExperimentAction.created_at)).first()

            ## keep eligible True if there's no previous action
            ## if there is a previous action, check that it falls within
            ## the interval
            if(last_experiment_action):
                last_experiment_created = last_experiment_action.created_at.replace(tzinfo=pytz.utc)
                interval_since_last_action = (current_time - last_experiment_created).total_seconds()
                eligible = ((interval_since_last_action < self.experiment_settings['intervention_interval_seconds'] + self.experiment_settings['intervention_window_seconds']) and (interval_since_last_action >= self.experiment_settings['intervention_interval_seconds'] - self.experiment_settings['intervention_window_seconds']))
                ## if more than 36 hours have elapsed, then we're eligible
                if(interval_since_last_action>129600):
                    eligible = True
        if(eligible==False):
            self.log.info("{0}: Experiment {1} code run. Ineligible to Continue. Begin Time: {2}. End Time: {3}. Last Intervention: {4} ".format(
                self.__class__.__name__,
                self.experiment.name,
                start_time,
                end_time,
                last_experiment_created
            ))

        return eligible

    ### TWO CONDITIONS: NORMAL AND SPECIAL
    ### MONDAYS FRIDAYS SATURDAYS
    def select_condition(self, current_time=datetime.datetime.utcnow()):
        isoweekdays = {7:"Sunday",1:"Monday", 2:"Tuesday",
            3:"Wednesday", 4:"Thursday", 5:"Friday", 6:"Saturday"}

        isoweekday = isoweekdays[current_time.date().isoweekday()]

        ## weekdays in the configuration file conditions should be exclusive
        for condition_name, values in self.experiment_settings['conditions'].items():
            if isoweekday in values['days'].split(","):
                return condition_name
        return None

    def run_intervention(self, condname):
        cond = self.experiment_settings['conditions'][condname]
        no_randomizations_remain = False

        try:
            randomization = cond['randomizations'][cond['next_randomization']]
            arm = "arm_" + randomization['treatment']
            self.experiment_settings['conditions'][condname]['next_randomization'] += 1
        except:
            #import pdb;pdb.set_trace
            self.log.error("{0}: Experiment {1} condition {2} has used its full stock of {3} {4} conditions. Cannot assign any further.".format(
                self.__class__.__name__,
                self.experiment.name,
                condname,
                len(cond['randomizations']),
                arm
            ))
            no_randomizations_remain = True

        if(no_randomizations_remain):
            return False

        self.experiment.settings_json = json.dumps(self.experiment_settings)
        self.db_session.commit()
        self.log.info("{0}: Experiment {1}: assigned condition {2} arm {3}".format(
            self.__class__.__name__, self.experiment.name, condname, arm))

        intervene = getattr(self, "intervene_" + condname + "_" + arm)
        return intervene(condname)

    def set_stylesheet(self, condition, arm):
        arms = self.experiment_settings['conditions'][condition]['arms']
        intervention_line = arms[arm]

        found_code = False

        stylesheet_data = self.r.get_stylesheet(self.subreddit)
        if "stylesheet" in stylesheet_data.keys():
            line_list = []
            for line in stylesheet_data['stylesheet'].split("\n"):
                ## IF A LINE FROM THE STUDY IS FOUND,
                ## REPLACE IT WITH THE INTERVENTION
                if line in arms.values():
                    line = intervention_line
                    found_code = True
                line_list.append(line)

            ## IF THE CODE IS NOT FOUND, ADD IT TO THE END
            if(found_code!=True):
                line_list.append("/* CivilServantBot Experiment CSS */")
                line_list.append(intervention_line)
                line_list.append("")
            new_stylesheet = "\n".join(line_list)

        result = self.r.set_stylesheet(self.subreddit, new_stylesheet)
        if('errors' in result.keys() and len(result['errors'])==0):

            self.log.info("{0}: Experiment {1}: Applied Arm {3} of Condition {4} in {2}".format(
                    self.__class__.__name__,
                    self.experiment.id,
                    self.subreddit,
                    arm,
                    condition))
            experiment_action = ExperimentAction(
                experiment_id = self.experiment.id,
                praw_key_id = PrawKey.get_praw_id(ENV, self.experiment_name),
                action = "Intervention",
                action_object_type = ThingType.STYLESHEET.value,
                action_object_id = None,
                metadata_json  = json.dumps({"arm":arm, "condition":condition})

            )
            self.db_session.add(experiment_action)
            self.db_session.commit()
        else:
            self.log.error("{0}: Experiment {1}: Failed to apply Arm {2} of Condition {3}. Reddit errors: {4}".format(
                    self.__class__.__name__,
                    self.experiment.id,
                    self.subreddit,
                    arm,condition, ", ".join(result['errors'])))
#            experiment_action = ExperimentAction(
#                experiment_id = self.experiment.id,
#                praw_key_id = PrawKey.get_praw_id(ENV, self.experiment_name),
#                action = "NonIntervention:PrawError.{0}.{1}".format(condition,arm),
#                action_object_type = ThingType.STYLESHEET.value,
#                action_object_id = None
#            )
#            self.db_session.add(experiment_action)
#            self.db_session.commit()
            ## IF WE FAILED TO APPLY THE INTERVENTION, ROLL BACK THAT RANDOMIZATION
            self.experiment_settings['conditions'][condname]['next_randomization'] -= 1
            self.experiment.settings_json = json.dumps(self.experiment_settings)
            self.db_session.commit()

        ## TO HELP WITH TESTING, RETURN THE FULL TEXT OF THE STYLESHEET
        return new_stylesheet

    def intervene_normal_arm_0(self, condname):
        return self.set_stylesheet(condname, "arm_0")

    def intervene_normal_arm_1(self, condname):
        return self.set_stylesheet(condname, "arm_1")

    def intervene_special_arm_0(self, condname):
        return self.set_stylesheet(condname, "arm_0")

    def intervene_special_arm_1(self, condname):
        return self.set_stylesheet(condname, "arm_1")

    ######################
    ## COMMENT SNAPSHOTS
    ######################

    ## THIS HIGH LEVEL METHOD TAKES A SNAPSHOT OF COMMENTS THAT NEED SAMPLING
    ## All
    def archive_experiment_submission_metadata(self):
        posts = self.identify_posts_that_need_snapshotting()
        comments = self.sample_comments(posts)
        self.observe_comment_snapshots(comments)

    ## IDENTIFY POSTS AND ALSO CREATE AN EXPERIMENT_THING
    ## FOR POSTS THAT DON'T YET HAVE ONE
    def identify_posts_that_need_snapshotting(self):
        last_action = self.db_session.query(ExperimentAction).filter(
            ExperimentAction.experiment_id == self.experiment.id,
            ExperimentAction.action=="Intervention").order_by(
            ExperimentAction.created_at
            ).first()
        if(last_action is None):
          return []
        eligible_posts = []
        for post in self.db_session.query(Post).filter(
            Post.created_at >= last_action.created_at,
            Post.subreddit_id == self.experiment_settings['subreddit_id']).all():
            eligible_posts.append(post)

        # find posts that are unpaired
        added_experiment_things = 0
        ## in th future,
        # use the post prefix for ExperimentThing indices because
        # the ExperimentThing table includes other object types
        # and there might be collisions
        #post_prefix = "t3_"
        for post in self.db_session.query(Post).outerjoin(
            ExperimentThing, Post.id == ExperimentThing.id).filter(
            ExperimentThing.id==None,
            Post.id.in_([x.id for x in eligible_posts])).all():

            et = ExperimentThing(
              id = post.id,
              object_type = ThingType.SUBMISSION.value,
              experiment_id = self.experiment.id,
              object_created = post.created,
              metadata_json = last_action.metadata_json
            )
            self.db_session.add(et)
            added_experiment_things += 1
        self.db_session.commit()

        self.log.info("{0}: Experiment {1}: Added {2} posts for comment monitoring in r/{3}.".format(
                self.__class__.__name__,
                self.experiment.id,
                added_experiment_things,
                self.subreddit))

        return eligible_posts

    ## THIS METHOD CHOOSES COMMENTS TO SAMPLE
    ## COMMENTS ARE SELECTED IF THEY'RE TOPLEVEL COMMENTS
    ## AND ADDS THEM AS EXPERIMENT_THINGS FOR LATER SNAPSHOTTING
    def sample_comments(self, posts):
        #posts = self.identify_posts_that_need_snapshotting()

        comment_things_to_observe = []
        comments_to_observe = []

        # STEP ONE: FOR EACH POST,
        # FIND OUT HOW MANY EXPERIMENT_THINGS ARE ASSOCIATED WITH THAT POST
        posts_needing_comments = defaultdict(list)
        comment_thing_counts = []

        for post in posts:
            comment_things = list(self.db_session.query(ExperimentThing).filter(
                           ExperimentThing.query_index == post.id,
                           ExperimentThing.experiment_id == self.experiment.id).all())
            comment_things_to_observe = comment_things_to_observe + comment_things
            comment_thing_counts.append(len(comment_things))

            if(len(comment_things) < self.experiment_settings['first_n_comments']):
                posts_needing_comments[post.id] = comment_things

        # this shouldn't be more than tens of thousands of comments
        # for an experiment that randomizes on a day basis.
        # In other experiments, it might be important to query on a per-post basis
        post_comments = defaultdict(list)
        added_n_comments_for_monitoring = 0
        for comment in self.db_session.query(Comment).filter(Comment.post_id.in_([x for x in posts_needing_comments.keys()])).order_by(asc(Comment.created_utc)):

            post_comments[comment.post_id].append(comment)

        for post_id, comments in post_comments.items():
            already_observing = [x.id for x in posts_needing_comments[post_id]]
            post_comment_count = len(already_observing)
            for comment in comments:
                # if the comment is toplevel and hasn't been seen before
                # and we're under our quota, then add an experiment_thing.
                # snapshots will be taken in a separate method
                if(post_comment_count < self.experiment_settings['first_n_comments'] and
                   comment.post_id == comment.post_id and comment.id not in already_observing):

                    comments_to_observe.append(comment)

                    et = ExperimentThing(
                      id = comment.id,
                      object_type = ThingType.COMMENT.value,
                      experiment_id = self.experiment.id,
                      object_created = comment.created_utc,
                      query_index = post_id
                    )
                    self.db_session.add(et)
                    post_comment_count += 1
                    added_n_comments_for_monitoring += 1

        self.log.info("{0}: Experiment {1}: Added {2} comments for monitoring in r/{3}".format(
                self.__class__.__name__,
                self.experiment.id,
                added_n_comments_for_monitoring,
                self.subreddit))

        self.db_session.commit()

        ## now fetch the remaining comments
        if(len(comment_things_to_observe)>0):
            comments_to_observe = comments_to_observe + list(self.db_session.query(Comment).filter(Comment.id.in_([x.id for x in comment_things_to_observe])).all())

        return comments_to_observe

    ## THIS METHOD OBSERVES POSTS IN THE EXPERIMENT PERIOD THAT DON'T HAVE
    ## AN EXPERIMENT THING RECORD OR HAVE ONE BUT NOT AN EXPERIMENT ACTION. IT THEN:
    ## - ASSIGNS THEM AN EXPERIMENT THNG
    ## - FOR POSTS WITH EXPERIMENT THINGS, WHICH HAVEN'T HAD
    ##   A CommentSampleComplete EXPERIMENT_ACTION
    ## - RUN THE observe_first_comments method
    ## - IF THE FIRST N COMMENTS ARE FOUND, ADD A
    ##   CommentSampleComplete EXPERIMENT_ACTION
    def observe_comment_snapshots(self, comments_to_observe):
        current_time = datetime.datetime.utcnow()
        intervention_window = self.experiment_settings['comment_snapshot_period_seconds']
        eligible_comment_ids = [x.id for x in comments_to_observe
                             if (current_time - x.created_utc).total_seconds() < intervention_window]
        #comment_things = self.db_session.query(ExperimentThing).filter(ExperimentThing.id.in_(eligible_comment_ids)).all()
        reddit_comment_ids = ["t1_" + x for x in eligible_comment_ids]
        if(len(reddit_comment_ids) == 0):
            self.log.info("{0}: Experiment {1}: Collected Snapshots from 0 comments in r/{3}.".format(
                    self.__class__.__name__,
                    self.experiment.id,
                    len(reddit_comment_ids),
                    self.subreddit))
            return
        for comment in self.r.get_info(thing_id = reddit_comment_ids):
            snapshot = {"score":comment.score,
                        "num_reports":comment.num_reports,
                        "user_reports":len(comment.user_reports),
                        "ups":comment.ups,
                        "downs":comment.downs,
                        "mod_reports":len(comment.mod_reports)
                        }
            experiment_thing_snapshot = ExperimentThingSnapshot(
                experiment_thing_id = comment.id,
                object_type = ThingType.COMMENT.value,
                experiment_id = self.experiment.id,
                metadata_json = json.dumps(snapshot)
            )
            self.db_session.add(experiment_thing_snapshot)
        self.db_session.commit()

        self.log.info("{0}: Experiment {1}: Collected Snapshots from {2} comments in r/{3}.".format(
                self.__class__.__name__,
                self.experiment.id,
                len(reddit_comment_ids),
                self.subreddit))


text:
using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Web.Mvc;

namespace CompareCloudware.Web.Models
{
    public class ContentPageModel : BaseModel
    {
        public int ContentPageID { get; set; }
        public string Slug { get; set; }
        public string Title { get; set; }
        public string Author { get; set; }
        public string GooglePlusId { get; set; }
        public float SortOrder { get; set; }
        public string Keywords { get; set; }
        public string Markdown { get; set; }
        public DateTime Modified { get; set; }
        public bool NoSearch { get; set; }
        public short Frequency { get; set; }
        public double? Priority { get; set; }

        public string MetaTitle { get; set; }
        public string MetaKeywords { get; set; }
        public string MetaDescription { get; set; }

        public string Route { get; set; }
    }
}


text:
import React from 'react'
import { shallow } from 'enzyme'

import Header from '../index'

describe('<Header />', () => {
  it('should render a styled header called Wrapper', () => {
    const renderedComponent = shallow(
      <Header />
    )
    expect(renderedComponent.find('Wrapper').length).toEqual(1)
  })

  it('should render 3 Links', () => {
    const renderedComponent = shallow(
      <Header />
    )
    expect(renderedComponent.find('Link').length).toEqual(3)
  })

  describe('toggleNav', () => {
    it('should toggle state of expanded from false to true', () => {
      const renderedComponent = shallow(
        <Header />
      )
      expect(renderedComponent.state('expanded')).toBe(false)
      renderedComponent.instance().toggleNav()
      expect(renderedComponent.state('expanded')).toBe(true)
    })

    it('should toggle state of expanded from true to false', () => {
      const renderedComponent = shallow(
        <Header />
      )
      expect(renderedComponent.state('expanded')).toBe(false)
      renderedComponent.setState({ expanded: true })
      expect(renderedComponent.state('expanded')).toBe(true)
      renderedComponent.instance().toggleNav()
      expect(renderedComponent.state('expanded')).toBe(false)
    })
  })

  describe('handleKeyDown', () => {
    it('should toggle expanded to false if escape key is pressed and expanded is true', () => {
      const renderedComponent = shallow(
        <Header />
      )
      renderedComponent.setState({ expanded: true })
      expect(renderedComponent.state('expanded')).toBe(true)
      renderedComponent.instance().handleKeyDown({ keyCode: 27 })
      expect(renderedComponent.state('expanded')).toBe(false)
    })

    it('should not toggle expanded to false if key pressed is not escape', () => {
      const renderedComponent = shallow(
        <Header />
      )
      expect(renderedComponent.state('expanded')).toBe(false)
      renderedComponent.instance().handleKeyDown({ keyCode: 26 })
      expect(renderedComponent.state('expanded')).toBe(false)
    })

    it('should not toggle expanded to false if escape key is pressed and expanded is false', () => {
      const renderedComponent = shallow(
        <Header />
      )
      expect(renderedComponent.state('expanded')).toBe(false)
      renderedComponent.instance().handleKeyDown({ keyCode: 27 })
      expect(renderedComponent.state('expanded')).toBe(false)
    })
  })

  describe('setExpandedFalse', () => {
    it('should set expanded to false if true', () => {
      const renderedComponent = shallow(
        <Header />
      )
      renderedComponent.setState({ expanded: true })
      expect(renderedComponent.state('expanded')).toBe(true)
      renderedComponent.instance().setExpandedFalse()
      expect(renderedComponent.state('expanded')).toBe(false)
    })

    it('should not set expanded to false if already false', () => {
      const renderedComponent = shallow(
        <Header />
      )
      expect(renderedComponent.state('expanded')).toBe(false)
      renderedComponent.instance().setExpandedFalse()
      expect(renderedComponent.state('expanded')).toBe(false)
    })
  })
})


text:
/************************************************************************/
/* Copyright (c) 2006 Cluracan (strangelove@heroinpuppy.com)            */
/*                                                                      */
/* The "MIT" License                                                    */
/*                                                                      */
/* Permission is hereby granted, free of charge, to any person          */
/* obtaining a copy of this software and associated documentation       */
/* files (the "Software"), to deal in the Software without restriction, */
/* including without limitation the rights to use, copy, modify, merge, */
/* publish, distribute, sublicense, and/or sell copies of the Software, */
/* and to permit persons to whom the Software is furnished to do so,    */
/* subject to the following conditions:                                 */
/*                                                                      */
/* - The above copyright notice and this permission notice shall be     */
/* included in all copies or substantial portions of the Software.      */
/*                                                                      */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,      */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES      */
/* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND             */
/* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT          */
/* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,         */
/* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,   */
/* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER        */
/* DEALINGS IN THE SOFTWARE.                                            */
/************************************************************************/

#include "net.h"
#include "pfx_heap.h"
#include "misc.h"
#include "xfer_c2s_enc_stock.h"

namespace yiffurc {

const XFER_C2S_ENC_STOCK xfer_c2s_enc_stock;

const char* YIFFURC_CALL xfer_c2s_enc_atomicfetch( const char* player,
                                                const char* pass,
                                                const char* package,
                                                size_t start_off,
                                                unsigned int send_ahead,
                                                size_t chunk_size )
{
   size_t player_len     = STRLEN( player );
   size_t pass_len       = STRLEN( pass );
   size_t package_len    = STRLEN( package );
   size_t start_off_len  = dec_len( (unsigned int) start_off );
   size_t send_ahead_len = dec_len( send_ahead );
   size_t chunk_size_len = dec_len( (unsigned int) chunk_size );

   char* out = pfx_new<char>( 3 + player_len + 1 + pass_len +
                              1 + package_len + 1 + start_off_len +
                              1 + send_ahead_len + 1 + chunk_size_len + 2 );
   const char* const ret = out;

   out = chain_memcpy( out, "RC ", 3 );
   out = chain_memcpy( out, player, player_len );
   *out = ' ';  out++;
   out = chain_memcpy( out, pass, pass_len );
   *out = ' ';  out++;
   out = chain_memcpy( out, package, package_len );
   *out = ' ';  out++;
   out = chain_itoa( out, (unsigned int) start_off );
   *out = ' ';  out++;
   out = chain_itoa( out, send_ahead );
   *out = ' ';  out++;
   out = chain_itoa( out, (unsigned int) chunk_size );
   out = chain_memcpy( out, "\n", 2 );

   return ret;
}

const unsigned char* YIFFURC_CALL xfer_c2s_enc_chunkhead( unsigned int chunk_id, unsigned int chunk_crc32 )
{
   chunk_id    = machine_to_lsb32( chunk_id );
   chunk_crc32 = machine_to_lsb32( chunk_crc32 );

   unsigned char* out = (unsigned char*) pfx_new<char>( 2 + sizeof( unsigned int ) + sizeof( unsigned int ) );
   const unsigned char* const ret = out;

   *out = 'S'; out++;
   *out = 'C'; out++;
   *(unsigned int*)out = chunk_id; out += sizeof( unsigned int );
   *(unsigned int*)out = chunk_crc32; out += sizeof( unsigned int );

   return ret;
}

const char* YIFFURC_CALL xfer_c2s_enc_password( const char* pass )
{
   size_t pass_len = STRLEN( pass );

   char* out = pfx_new<char>( 5 + pass_len + 2 );
   const char* const ret = out;

   out = chain_memcpy( out, "PASS ", 5 );
   out = chain_memcpy( out, pass, pass_len );
   out = chain_memcpy( out, "\n", 2 );

   return ret;
}

const char* YIFFURC_CALL xfer_c2s_enc_setr( int unk )
{
   size_t unk_len = dec_len( unk );

   char* out = pfx_new<char>( 5 + unk_len + 2 );
   const char* const ret = out;

   out = chain_memcpy( out, "SETR ", 5 );
   out = chain_itoa( out, unk );
   out = chain_memcpy( out, "\n", 2 );

   return ret;
}

const char* YIFFURC_CALL xfer_c2s_enc_upload2pkg( unsigned int dream_id, int unk )
{
   size_t dream_id_len = dec_len( dream_id );
   size_t unk_len      = dec_len( unk );

   char* out = pfx_new<char>( 8 + dream_id_len + 1 + unk_len + 2 );
   const char* const ret = out;

   out = chain_memcpy( out, "SNCSEND ", 8 );
   out = chain_itoa( out, dream_id );
   *out = ' '; out++;
   out = chain_itoa( out, unk );
   out = chain_memcpy( out, "\n", 2 );

   return ret;
}

const char* YIFFURC_CALL xfer_c2s_enc_uploadinfo( const char* filename,
                                               size_t filesize )
{
   size_t filename_len = STRLEN( filename );
   size_t filesize_len = dec_len( (unsigned int) filesize );

   char* out = pfx_new<char>( 5 + filename_len + 1 + filesize_len + 2 );
   const char* const ret = out;

   out = chain_memcpy( out, "SEND ", 5 );
   out = chain_memcpy( out, filename, filename_len );
   *out = ' '; out++;
   out = chain_itoa( out, (unsigned int) filesize );
   out = chain_memcpy( out, "\n", 2 );

   return ret;
}

const char* YIFFURC_CALL xfer_c2s_enc_user( const char* player )
{
   size_t player_len = STRLEN( player );

   char* out = pfx_new<char>( 5 + player_len + 2 );
   const char* const ret = out;

   out = chain_memcpy( out, "USER ", 5 );
   out = chain_memcpy( out, player, player_len );
   out = chain_memcpy( out, "\n", 2 );

   return ret;
}

const char* YIFFURC_CALL xfer_c2s_enc_bye( )
{
   return xfer_c2s_enc_stock.sz_bye;
}

} // namespace yiffurc


text:
/* @flow */

module.exports = {
  get withNativeAd() {
    return require('./withNativeAd').default;
  },
  get AdSettings() {
    return require('./AdSettings').default;
  },
  get NativeAdsManager() {
    return require('./NativeAdsManager').default;
  },
  get InterstitialAdManager() {
    return require('./InterstitialAdManager').default;
  },
  get BannerView() {
    return require('./BannerViewManager').default;
  },
};


text:
'use strict';

/**
 * Describes the type of token data being sent to connected clients
 */
export enum ClientInstructionType {
    /**
     * Event sent to clients whenever a user logs in.
     * Event type: IUserToken
     */
    Login = 1,

    /**
     * Event sent to clients whenever a user logs out.
     * Event type: IUserToken
     */
    Logout = 2,

    /**
     * Event sent to clients whenever a user's account is activated.
     * Event type: IUserToken
     */
    Activated = 3,

    /**
     * Event sent to clients whenever a user's account is removed.
     * Event type: IUserToken
     */
    Removed = 4,

    /**
     * Event sent to clients whenever a user uploads a new file.
     * Event type: IFileToken
     */
    FileUploaded = 5,

    /**
     * Event sent to clients whenever a user file is removed.
     * Event type: IFileToken
     */
    FileRemoved = 6,

    /**
     * Event sent to clients whenever a user creates a new bucket
     * Event type: IBucketToken
     */
    BucketUploaded = 7,

    /**
     * Event sent to clients whenever a user removes a bucket
     * Event type: IBucketToken
     */
    BucketRemoved = 8,

    /**
     * Event both sent to the server as well as optionally to clients. Gets or sets user meta data.
     * Event type: IMetaToken
     */
    MetaRequest = 9
}

/**
 * Describes the type of token data being sent to connected clients
 */
export enum ServerInstructionType {
    /**
     * Event both sent to the server as well as optionally to clients. Gets or sets user meta data.
     * Event type: IMetaToken
     */
    MetaRequest = 9
}


text:
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">

<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<meta name="generator" content="Doxygen 1.8.6"/>
<title>Face Detection with Alignment: eHposemodel.h File Reference</title>
<link href="tabs.css" rel="stylesheet" type="text/css"/>

<link href="search/search.css" rel="stylesheet" type="text/css"/>

<link href="doxygen.css" rel="stylesheet" type="text/css" />

Face Detection with Alignment

Main Page

Classes

Files

          <input type="text" id="MSearchField" value="Search" accesskey="S"
               onfocus="searchBox.OnSearchFieldFocus(true)"
               onblur="searchBox.OnSearchFieldFocus(false)"
               onkeyup="searchBox.OnSearchFieldChange(event)"/>

File List

File Members

 All Classes Files Functions Variables Typedefs Macros

Classes |
Typedefs |
Functions

eHposemodel.h File Reference

Human body/pose detection model and operations.
More...

#include <vector>

#include "eHimage.h"

#include "eHfilter.h"

#include "eHbbox.h"

Go to the source code of this file.

Classes

struct  	eHposemodel

 	Human body/pose mdoel.  More...

Typedefs

typedef struct deformation_pose 	posedef_t

typedef struct part_pose 	posepart_t

typedef struct bias_pose 	posebias_t

typedef struct eHposemodel 	posemodel_t

Functions

posemodel_t * 	posemodel_parseXml (char *xmlstr)

 	Parse body/pose model from xml style string.  More...

posemodel_t * 	posemodel_readFromFile (const char *filepath)

 	Read body/pose model from file.  More...

vector< bbox_t > 	posemodel_detect (const posemodel_t *model, const image_ptr img, double thrs)

 	Perform body/pose detection.  More...

vector< bbox_t > 	posemodel_detect (const posemodel_t *model, const image_ptr img)

 	Perform body/pose detection using default threshold.  More...

void 	posemodel_delete (posemodel_t *model)

 	Delete a pose model, release related memory.

Detailed Description

Human body/pose detection model and operations.

See Also

Y. Yang, D. Ramanan, "Articulated Pose Estimation using Flexible Mixtures of Parts". In CVPR 2011.

Author

Hang Su

Date

2012-08

Function Documentation

          vector<bbox_t> posemodel_detect
          (
          const posemodel_t *
          model,

          const image_ptr
          img,

          double
          thrs

          )

Perform body/pose detection.

Parameters

model	pose detection model

img	where to find human poses from

thrs	threshold used for pruning results

Returns

array of detected poses (together with part locations)

          vector<bbox_t> posemodel_detect
          (
          const posemodel_t *
          model,

          const image_ptr
          img

          )

Perform body/pose detection using default threshold.

Parameters

model	pose detection model

img	where to find human poses from

Returns

array of detected poses (together with part locations)

          posemodel_t* posemodel_parseXml
          (
          char *
          xmlstr	)

Parse body/pose model from xml style string.

Note

xmlstr is modified during parsing, this can be avoided by using Non-Destrutive Mode of rapidxml

          posemodel_t* posemodel_readFromFile
          (
          const char *
          filepath	)

Read body/pose model from file.

See Also

posemodel_parseXml()

<address class="footer">
Generated on Mon May 26 2014 22:18:57 for Face Detection with Alignment by

 1.8.6
</address>


text:
set -o nounset -o pipefail -o errexit -x

genome=~astlingd/genomes/bowtie-1.1.2/hg38/hg38
fastq_folder=fastq/replicates
result_folder=alignments/replicates/total-strata

SAMPLES=( $fastq_folder/*_1.fastq.gz )
sample=${SAMPLES[$(($LSB_JOBINDEX - 1))]}

sample=`echo $sample | perl -pe 's/^.+?\/([^\/]+?)_[12].fastq.gz/$1/'`
first_pair=$fastq_folder/${sample}_1.fastq
second_pair=$fastq_folder/${sample}_2.fastq
bam=$result_folder/${sample}_paired.bam

gunzip -c $first_pair.gz > $first_pair
gunzip -c $second_pair.gz > $second_pair

code/bowtie_total.sh -g $genome -b $bam $first_pair $second_pair

bam=$result_folder/${sample}_single.bam
code/bowtie_multiread.sh -g $genome -b $bam $first_pair

#bowtie -p 12 -X 2000 --best --strata --all -v 2 -S --un $result_folder/${sample}_paired_unaligned $genome \
#-1 $first_pair -2 $second_pair | samtools view -Sb - > $result_folder/${sample}_paired.bam

rm $first_pair
rm $second_pair


text:
#include "third_party/blink/renderer/core/svg/svg_animated_path.h"

#include "third_party/blink/renderer/core/css/css_identifier_value.h"
#include "third_party/blink/renderer/platform/heap/garbage_collected.h"

namespace blink {

SVGAnimatedPath::SVGAnimatedPath(SVGElement* context_element,
                                 const QualifiedName& attribute_name,
                                 CSSPropertyID css_property_id)
    : SVGAnimatedProperty<SVGPath>(context_element,
                                   attribute_name,
                                   MakeGarbageCollected<SVGPath>(),
                                   css_property_id) {}

SVGAnimatedPath::~SVGAnimatedPath() = default;

const CSSValue& SVGAnimatedPath::CssValue() const {
  const cssvalue::CSSPathValue& path_value = CurrentValue()->PathValue();
  if (path_value.GetStylePath()->ByteStream().IsEmpty())
    return *CSSIdentifierValue::Create(CSSValueID::kNone);
  return path_value;
}

}  // namespace blink


text:
from django.core.management.base import BaseCommand
from django.conf import settings
import json
import requests
from time import sleep
from tracking import utils_freshdesk
from tracking.utils import logger_setup

class Command(BaseCommand):
    help = 'Download and cache recent Freshdesk tickets.'

    def add_arguments(self, parser):
        # Named (optional) arguments:
        parser.add_argument(
            '-l',
            '--limit',
            action='store',
            dest='limit',
            default=0,  # No limit.
            type=int,
            help='Maximum number of tickets to download and cache.',
        )

    def handle(self, *args, **options):
        logger_headers = logger_setup('freshdesk_api_response_headers')
        # Begin by caching Agents as Contacts.
        utils_freshdesk.freshdesk_cache_agents()
        # Next, start caching tickets one page at a time.
        url = settings.FRESHDESK_ENDPOINT + '/tickets'
        # By default, the 'list tickets' API returns tickets created in the
        # past 30 days only. If older tickets need to be cached, modify the
        # params dict below to include a value for "updated_since".
        # Ref: https://developer.freshdesk.com/api/#list_all_tickets
        params = {'page': 1, 'per_page': 100}
        further_results = True
        cached_count = 0

        while further_results:
            if options['limit'] and (cached_count + params['per_page']) >= options['limit']:
                params['per_page'] = options['limit'] - cached_count
            r = requests.get(url, auth=settings.FRESHDESK_AUTH, params=params)
            logger_headers.info(json.dumps(dict(r.headers)))
            # If we've been rate-limited, response status will be 429.
            # Sleep for the number of seconds specifief by the Retry-After header.
            if r.status_code == 429:
                if 'retry-after' in r.headers:
                    naptime = r.headers['retry-after']
                else:
                    naptime = 3600  # Sleep for an hour.
                sleep(naptime)
            # If the response
            elif r.status_code == 200:
                if 'link' not in r.headers:  # No further paginated results.
                    further_results = False
                else:
                    params['page'] += 1
                tickets = r.json()
                cache = utils_freshdesk.freshdesk_cache_tickets(tickets)
                if not cache:  # Error!
                    further_results = False
                cached_count += len(tickets)
                if options['limit'] and cached_count >= options['limit']:
                    print('Caching limit reached; terminating.')
                    further_results = False
            else:
                further_results = False