<file_sep>class Chunk(object):
  '''
  Class used to store and manage generated chunk data/metadata
  '''
  def __init__(self,Schema,chunk_id,cellcount,coordinates,attributes):
    self.schema = Schema
    self.chunk_id = chunk_id
    self.coordinates = coordinates
    self.attributes = attributes
    self.cellcount = cellcount
<file_sep>from scidb_attribute import ScidbAttribute
from scidb_dimension import ScidbDimension
import math

class ScidbSchema:
  '''
  Class used to store a parsed SciDB schema.
  '''
  def __init__(self, name, attributes, dimensions):
    '''
    Arguments:
      name       -- name of the array
      attributes -- list of ScidbAttribute objects
      dimensions -- list of ScidbDimension objects
    '''
    self.name = ""
    self.attributes = []
    self.dimensions = []

    if name is not None:
      self.name = name
    if (attributes is not None) and (len(attributes) > 0):
      self.attributes = attributes
    if (dimensions is not None) and (len(dimensions) > 0):
      self.dimensions = dimensions

  def compute_dimid(self,chunk_id,cell_id):
    '''
    Given a chunk id and local cell id for the chunk,
    this method uses the schema definition to generate proper dimension id's for the
    cell.

    Arguments:
      chunk_id -- a single integer representing the location of the chunk.
      cell_id -- a single integer representing the location of the cell within the given chunk
    
    Returns:
      An integer array of length equal to the number of dimensions in the schema
    Note: I assume the first dimension is most significant, and last dimension least significant
    '''
    numdims = len(self.dimensions)
    base_index = [0] * numdims
    offset_index = [0] * numdims
    chunksize = self.compute_chunksize()
    totalchunks = self.compute_totalchunks()
    range_starts = []
    base_widths = []
    widths = []
    for dim in self.dimensions:
      chunks = math.ceil((1.0*dim.range_end - dim.range_start + 1) / dim.chunk_width)
      base_widths.append(chunks) # chunks along this dimension
      widths.append(dim.chunk_width)
      range_starts.append(dim.range_start)
    if numdims > 1:
      for i,base_width in enumerate(base_widths[:numdims-1]):
        totalchunks /= base_width
        while chunk_id >= totalchunks:
          chunk_id -= totalchunks
          base_index[i] += 1
        base_index[i] *= widths[i] # want the logical index, not chunk index

      for i,width in enumerate(widths[:numdims-1]):
        chunksize /= width
        while cell_id >= chunksize:
          cell_id -= chunksize
          offset_index[i] += 1
        offset_index[i] += base_index[i] + range_starts[i]# compute final index for this dim
    #final index for last dim
    base_index[numdims-1] = chunk_id * widths[numdims-1]
    offset_index[numdims-1] = cell_id + base_index[numdims-1] + range_starts[numdims - 1]
    return offset_index

  def compute_chunksize(self):
    '''
    Multiplies all chunk widths together to get the logical size of a single chunk in the schema
    '''
    chunksize = 1
    for dim in self.dimensions:
      chunksize *= dim.chunk_width
    return chunksize


  def compute_totalchunks(self):
    '''
    Computes the total number of chunks created in the schema by computing number of chunks along
    each dimension and multiplying them together.
    '''
    totalchunks = 1
    for dim in self.dimensions: # assume these are integers
      cw = dim.chunk_width
      rs = dim.range_start
      re = dim.range_end
      dimrange = re - rs + 1
      numchunks = math.ceil(dimrange / cw)
      if numchunks > 0:
        totalchunks *= numchunks
    return int(totalchunks)


  @staticmethod
  def parse_schema(schema_def):
    '''
    Method to parse a SciDB schema definition. Dimensions are assumed to be the default SciDB
    datatype of int64.
    Arguments:
      schema_def -- string representing a SciDB schema definiton
    Returns:
      tuple containing the name, list of attributes, and list of dimensions represented in schema_def

    Example:
      sdef = "array<attribute1:double>[dimension1=1:10,10,0,dimension2=1:10,10,0]"
      (name,attributes,dimensions) = Schema.parse(sdef)
    '''
    a1 = schema_def.index('<')
    a2 = schema_def.index('>')
    d1 = schema_def.index('[')
    d2 = schema_def.index(']')

    name = schema_def[:a1]

    attr_string = schema_def[a1+1:a2] # want to exclude '<'
    dim_string = schema_def[d1+1:d2] # want to exclude '['

    attributes = ScidbSchema.parse_attributes(attr_string)
    dimensions = ScidbSchema.parse_dimensions(dim_string)

    return (name,attributes,dimensions)

  @staticmethod
  def parse_attributes(attr_string):
    '''
    Turns a SciDB description of attributes into a list of ScidbAttribute objects
    Arguments:
      attr_string -- string representing attributes from a SciDB schema definition
                     *without* '<' and '>'
    Returns:
      A list of ScidbAttribute objects
    '''
    attribute_list = []

    #split on ','
    attrs = attr_string.split(',')
    #split each token on ':' for (name,dtype)
    for attr in attrs:
      pair = attr.split(':')
      if len(pair) == 2: # there should only be 2 items here
        name = pair[0]
        dtype = pair[1]
        attribute_list.append(ScidbAttribute(name,dtype))
      else:
        raise Exception("malformed attribute:\""+attr+"\"")
    return attribute_list   

  @staticmethod
  def parse_dimensions(dim_string):
    '''
    Turns a SciDB description of dimensions into a list of ScidbDimension objects
    Arguments:
      dim_string -- string representing dimensions from a SciDB schema definition
                     *without* '[' and ']'
    Returns:
      A list of ScidbDimension objects
    '''

    dimension_list = []
    #split on ','
    dims = dim_string.split(',')
    if (len(dims) % 3) != 0:
      raise Exception("Malformed dimension description. \
        Incorrect number of parameters per dimension: \""+dim_string+"\"")
    temp = None
    #traverse every 3 tokens (name+range,chunk width, chunk overlap)
    for i,dim in enumerate(dims):
      i = i % 3
      if i == 0: # name and dimension range
        temp = ScidbDimension(None,None,None,None,None,None)
        temp.dtype = "int64"
        #name+range
        nar = dim.split('=')
        if len(nar) == 2: # name and range only
          name = nar[0]
          if len(name) != 0:
            temp.name = name
          else: # name is empty
            raise Exception("Malformed dimension description. No dimension name:\""+dim+"\"")
          dim_range = nar[1].split(':')
          if (len(dim_range) == 2) and (len(dim_range[0]) > 0) and (len(dim_range[1]) > 0): 
            # range start and range end only
            temp.range_start = int(dim_range[0])
            temp.range_end = int(dim_range[1])
          else: # split for dimension range didn't work
            raise Exception("Malformed dimension description. Incorrect range:\""+dim+"\"")
        else: # split on '=' for name/range didn't work
          raise Exception("Malformed dimension description. \
            Typo with name/dimension range:\""+dim+"\"")
      elif i == 1: # chunk width
        if len(dim) > 0:
          temp.chunk_width = int(dim)
        else:
          raise Exception("Malformed dimension description. No chunk width.")
      else: # chunk overlap
        if len(dim) > 0:
          temp.chunk_overlap = int(dim)
          dimension_list.append(temp)
        else:
          raise Exception("Malformed dimension description. No chunk overlap.")       
    return dimension_list
<file_sep>import os
import numpy as np

offset = 0
offsetB = 0

def file_setup(Schema,aorb='A',path_prefix=''):
  dim_handles = []
  attr_handles = []
  chunkmap_handle = None
  path = os.path.join(path_prefix,Schema.name+aorb)
  try: # make the directory for the data
    os.makedirs(path)
  except OSError:
    if not os.path.isdir(path):
      raise
  numdims = len(Schema.dimensions)
  for i,dim in enumerate(Schema.dimensions):
    f = os.path.join(path,str(i)+'.chunk')
    dim_handles.append(open(f,'w'))
  for i,attr in enumerate(Schema.attributes):
    f = os.path.join(path,str(i+numdims)+'.chunk')
    attr_handles.append(open(f,'w'))
  f = os.path.join(path,'chunkmap')
  chunkmap_handle = open(f,'w')
  return (dim_handles,attr_handles,chunkmap_handle)

def close_handle(handle):
  '''
  Closes a single file handle
  '''
  handle.close()

def close_handles(handles):
  '''
  Closes a list of file handles
  '''
  for i in range(len(handles)):
    handles[i].close()

def write_dimvals(chunk,dim_handles,chunkmap_handle,aorb='A'):
  global offset
  global offsetB
  chunk_id = chunk.chunk_id
  coords = chunk.coordinates
  #coords contains 1 list for each nonempty cell of length # dims
  for coord in coords:
    for i,handle in enumerate(dim_handles):
      handle.write(str(coord[i]))
      handle.write('\n')
  o = offset
  if aorb == 'B':
    o = offsetB
  chunkmap_handle.write(','.join([str(chunk_id),str(o),str(chunk.cellcount)]))
  chunkmap_handle.write('\n')
  # do not close handles!!!
  if aorb == 'A':
    offset += chunk.cellcount
  else:
    offsetB += chunk.cellcount

def reset_offset(aorb='A'):
  global offset
  if aorb == 'B':
    offsetB = 0
  else:
    offset = 0

def write_dimvals_binary(chunk,dim_handles,chunkmap_handle):
  global offset
  chunk_id = chunk.chunk_id
  coords = np.array(chunk.coordinates)
  coords = coords.T # take the transpose
  #write to dimension files
  for i,handle in enumerate(dim_handles): #each row of coords is all values for dim i
    handle.write(coords[i].tostring())
  # do not close handles!!!

def write_attrvals_binary(chunk,attr_handles):
  chunk_id = chunk.chunk_id
  attrvals = np.array(chunk.attributes)
  # attrvals contains 1 list for each attribute of length cellcount
  for i,handle in enumerate(attr_handles):
    handle.write(attrvals[i].tostring())
  # do not close handles!

def write_attrvals(chunk,attr_handles):
  chunk_id = chunk.chunk_id
  attrvals = chunk.attributes
  # attrvals contains 1 list for each attribute of length cellcount
  for i,handle in enumerate(attr_handles):
    for attr in attrvals[i]:
      handle.write(str(attr))
      handle.write('\n')
  # do not write to chunkmap!
  # do not close handles!

def write_chunk(chunk,dim_handles,attr_handles,chunkmap_handle,aorb='A'):
  #print "writing chunk"
  write_dimvals(chunk,dim_handles,chunkmap_handle,aorb=aorb)
  write_attrvals(chunk,attr_handles)

def write_chunk_binary(chunk,dim_handles,attr_handles,chunkmap_handle):
  #print "writing chunk"
  write_dimvals_binary(chunk,dim_handles,chunkmap_handle)
  write_attrvals_binary(chunk,attr_handles)
  
<file_sep>import math
import random
import numpy as np

c = 0.0
first = True

c2 = 0.0
carr = None

def zipf(alpha,n):
  global c
  global first

  if first:
    c = 0.0
    for i in range(1,n+1):
      c += 1.0 / math.pow(i,alpha)
    c = 1.0 / c
    first = False

  z = random.random()
  while z == 0.0: # random excludes 1.0 by default
    z = random.random()

  sum_prob = 0
  zipf_value = 0
  for i in range(1,n+1):
    sum_prob += c / math.pow(i,alpha)
    if sum_prob >= z:
      zipf_value = i
      break

  if (zipf_value < 1) or (zipf_value > n):
    raise Exception("Zipf value out of bounds")

  return zipf_value

def zipf_variable(alpha,n):
  return lambda: zipf2(alpha,n)

#broken
def zipf2(alpha,n):
  global carr
  global c2
  if carr is None:
    print "carr is None, calculating..."
    carr = np.zeros(n)
    temp = np.array(range(1,n+1)).astype(float)
    temp = np.power(temp,alpha)
    temp = 1.0 / temp
    c2 = 1.0 / np.sum(temp)
    #print "c2:",c2,",temp:",temp
    temp = c2 * temp
    carr[0] = temp[0]
    for i in range(1,n):
      carr[i] = carr[i-1] + temp[i]
      #print "carr["+str(i)+"]:",carr[i]
    print "done calculating carr"

  z = np.random.uniform(.0000001,1)
  for i in range(1,n+1):
    if carr[i-1] >= z:
      return i
      
<file_sep>from scidb_attribute import *
from scidb_dimension import *
from scidb_schema import *
<file_sep>from partition import Partition

class RangePartition(Partition):
  def __init__(self,Schema,range_starts,range_widths):
    for i in range(len(range_starts)):
      for j in range(len(range_starts[i])):
        range_starts[i][j] = int(range_starts[i][j])
    for i in range(len(range_widths)):
      for j in range(len(range_widths[i])):
        range_widths[i][j] = int(range_widths[i][j])

    super(RangePartition,self).__init__(Schema,range_starts,range_widths)

  def get_chunks(self,node_id):
    '''
    Returns the list of chunk id's associated with the given node
    using this partitioning scheme
    '''
    node_chunks = []
    if self.nodecount <= node_id: # this layout usees fewer nodes
      return node_chunks
    range_start = self.range_starts[node_id]
    range_width = self.range_widths[node_id]
    range_end = [0] * len(range_start)
    for i in range(len(range_start)):
      range_end[i] = range_start[i] + range_width[i] - 1
    totalchunks = self.Schema.compute_totalchunks()
    #chunksize = self.Schema.compute_chunksize
    numdims = len(self.Schema.dimensions)
    for i in range(totalchunks):
      coord = self.Schema.compute_dimid(i,0)
      #print "chunk:",i,",coord:",coord,",range start:",range_start,",range end:",range_end
      add = True
      for dim in range(numdims):
        if (coord[dim] < range_start[dim]) or (coord[dim] > range_end[dim]):
          add = False
          break
      if add:
        #print "adding chunk",i
        node_chunks.append(i)
    return node_chunks

  def compute_overlap(self,other,node_id):
    '''
    Returns the list of chunk id's that are shared between this partitioning scheme and another
    partitioning scheme for the given node
    Arguments:
      other -- an instance of the Partition class to compare with
      node_id -- the node to compare for overlap between the partitioning schemes
    '''
    overlap = []
    non_overlap1 = [] # only need to keep track of this one
    non_overlap2 = [] # just tracking this to be thorough
    node1_chunks = self.get_chunks(node_id)
    node2_chunks = other.get_chunks(node_id)

    len1 = len(node1_chunks)
    len2 = len(node2_chunks)
    i1 = 0
    i2 = 0
    while (i1 < len1) and (i2 < len2):
      val1 = node1_chunks[i1]
      val2 = node2_chunks[i2]
      if val1 == val2:
        overlap.append(val1)
        i1 += 1
        i2 += 1
      elif val1 < val2:
        i1 += 1
        non_overlap1.append(val1)
      else:
        i2 += 1
        non_overlap2.append(val2)
    while i1 < len1:
      val1 = node1_chunks[i1]
      non_overlap1.append(val1)
      i1 += 1
    while i2 < len2:
      val2 = node2_chunks[i2]
      non_overlap2.append(val2)
      i2 += 1
    return (overlap,non_overlap1,non_overlap2)

  @staticmethod
  def parse(partition_string):
    #split on |
    node_strings = partition_string.split('|')
    numnodes = len(node_strings)
    range_starts = []
    range_widths = []
    #for each item (one for each node):
    for node_string in node_strings:
      # split on :, first is node name, second is starts and ranges
      nar = node_string.split(':')
      node_id = int(nar[0])
      range_string = nar[1]
      # split second on ',' and count the items
      range_items = range_string.split(',')
      # if the number of items is not even, this is wrong
      lri = len(range_items)
      if (lri % 2) != 0:
        raise Exception("malformed ranges for node "+nar[0]+": "+range_string)
      # store the first half as range_starts
      half = lri/2 # this should just work bc it's even
      range_starts.append(range_items[:half])
      # store the second half as range_widths
      range_widths.append(range_items[half:])
    return (range_starts,range_widths)
<file_sep>import schema_parser as parser
from zipf import zipf_variable
import math
import random
import io # located in io.py
from chunk import Chunk
from partition import Partition #do I need this?
from range_partition import RangePartition
import numpy as np

schema_defs = [
"array1<attr1:double,attr2:int64>[dim1=1:200000,1000,0,dim2=1:200000,1000,0]"
,"array5<attr1:int64,attr2:int64>[dim1=1:100,25,0,dim2=1:100,25,0]"
,"array6<attr1:double,attr2:int64>[dim1=1:1000,50,0,dim2=1:1000,50,0]"
#,"array2<attr1:double,attr2:int64>[dim1=1:10,5,0,dim2=0:9,5,0]"
#,"array4<attr1:double,attr2:int64>[dim1=0:9,5,0,dim2=0:9,5,0]"
,"array3<attr1:int64>[dim1=1:100,10,0,dim2=1:100,10,0]"
]

partition_defs = {
# quarters
'quarters':"0:1,1,100000,100000\
|2:1,100001,100000,100000\
|1:100001,1,100000,100000\
|3:100001,100001,100000,100000"
# rows
,'rows':"0:1,1,50000,200000\
|1:50001,1,50000,200000\
|2:100001,1,50000,200000\
|3:150001,1,50000,200000"
# quarters
,'quarters2':"0:1,1,50,50\
|2:1,51,50,50\
|1:51,1,50,50\
|3:51,51,50,50"
# rows
,'rows2':"0:1,1,25,100\
|1:26,1,25,100\
|2:51,1,25,100\
|3:76,1,25,100"
,'quarters3':"0:1,1,500,500\
|2:1,501,500,500\
|1:501,1,500,500\
|3:501,501,500,500"
# rows
,'rows3':"0:1,1,250,1000\
|1:251,1,250,1000\
|2:501,1,250,1000\
|3:751,1,250,1000"
# quarters
,'quarters4':"0:1,1,50,50\
|2:1,51,50,50\
|1:51,1,50,50\
|3:51,51,50,50"
# rows
,'rows4':"0:1,1,30,100\
|1:31,1,20,100\
|2:51,1,30,100\
|3:81,1,20,100"

}

#partition_defs = [
##modis quarters
#"0:0,-1800000,-900000,20161,1800000,900000\
#|2:0,-1800000,0,20161,1800000,900001\
#|1:0,0,-900000,20161,1800001,900000\
#|3:0,0,0,20161,1800001,900001"
##modis rows
#,"0:0,-1800000,-900000,20161,3600001,450000\
#|1:0,-1800000,-450000,20161,3600001,450000\
#|2:0,-1800000,0,20161,3600001,450000\
#|3:0,-1800000,450000,20161,3600001,450001"
#]

#print "name:",name
#print "attributes:",attributes
#print "dimensions:",dimensions

DEFAULT_ALPHA = 1.
DEFAULT_CHUNKSIZE = 100
DEFAULT_CUTOFF = .8

min_chunksize = .05
max_chunksize = .5
z_range = 10

join_alignment = .5

DEFAULT_ZIPF = zipf_variable(DEFAULT_ALPHA,DEFAULT_CHUNKSIZE)

##############################
'''
myschema = schema_defs[0]
myrange1 = 'quarters'
myrange2 = 'rows'
'''

'''
myschema = schema_defs[1]
myrange1 = 'quarters2'
myrange2 = 'rows2'
'''

'''
myschema = schema_defs[2]
myrange1 = 'quarters3'
myrange2 = 'rows3'
'''

myschema = schema_defs[3]
myrange1 = 'quarters4'
myrange2 = 'rows4'

##############################

def get_hotcold_cellcounts(totalchunks,n,distribution=DEFAULT_ZIPF):
  hotchunks = []
  coldchunks = []
  cutoff = DEFAULT_CUTOFF * n
  for i in range(totalchunks):
    val = distribution()
    if val >= cutoff:
      hotchunks.append(val)
    else:
      coldchunks.append(val)
  return (hotchunks,coldchunks)

def build_ABchunks(Schema,chunk_id,countA,countB,chunksize,overlap=1.):
  maxcount = max(countA,countB)
  mincount = min(countA,countB)
  # build the maxchunk from scratch
  maxchunk = build_chunk(Schema,chunk_id,maxcount,chunksize)
  if maxcount == mincount: # they are the same, so stop here
    return (maxchunk,maxchunk)
  attrlen = len(Schema.attributes)
  minchunk = Chunk(Schema,chunk_id,mincount,[],[[]]*attrlen) # empty chunk for now
  minitems = choose_randk(range(maxcount),mincount) # copy random subset of the larger chunk

  # copy the selected cells from maxchunk to minchunk
  for i in minitems:
    minchunk.coordinates.append(maxchunk.coordinates[i])
    for j in range(attrlen):
      minchunk.attributes[j].append(maxchunk.attributes[j][i])
  if countA > countB:
    chunkA = maxchunk
    chunkB = minchunk
  else:
    chunkA = minchunk
    chunkB = maxchunk
  return (chunkA,chunkB)
  

def build_chunk(Schema,chunk_id,cellcount,chunksize):
  chosen = []
  coords = []   # coordinates of nonempty cells
  attrvals = [] # attribute values of nonempty cells
  chunk = Chunk(Schema,chunk_id,cellcount,[],[]) # empty chunk for now

  if chunksize == 1:
    return [0]
  candidates = range(chunksize)
  if cellcount == chunksize:
    return candidates
  chosen = choose_randk(candidates,cellcount)
  #generate dimension coordinates
  for i in chosen:
    coords.append(Schema.compute_dimid(chunk_id,i))
  chunk.coordinates = coords
  #generate attribute values
  attrvals = generate_attributes(Schema.attributes,cellcount)
  chunk.attributes = attrvals
  #print "chosen:",chosen
  #print "dimensions for cell",chosen[0],"of chunk",chunk_id,":",Schema.compute_dimid(chunk_id,chosen[0])
  #print "coords for chunk",chunk_id,":",coords
  #print "attributes for chunk",chunk_id,":",attrvals
  return chunk

def shuffle_firstk(candidates,k):
  '''
  shuffles the first k items, and returns the resulting list. this assumes k < len(candidates)
  '''
  l = len(candidates) - 1
  for i in range(k): # shuffle the first k items
    x = random.randint(i,l) #should be possible to stay in place
    temp = candidates[x]
    candidates[x] = candidates[i]
    candidates[i] = temp
  return candidates #return everything

def choose_randk(candidates,k):
  '''
  returns a list of k randomly selected items chosen from the given list
  '''
  l = len(candidates) - 1
  for i in range(k): # shuffle the first k items
    x = random.randint(i,l) #should be possible to stay in place
    temp = candidates[x]
    candidates[x] = candidates[i]
    candidates[i] = temp
  return candidates[:k]

def build_chunks(Schema,cellcounts,chunksize):
  for i,cellcount in enumerate(cellcounts):
    build_chunk(Schema,i,cellcount,chunksize)


def generate_attributes(attributes,cellcount,shift=3):
  result = []
  window = math.pow(10,3)
  numattrs = len(attributes)

  for attr in attributes:
    #array of length cellcount for each attribute
    x = []
    if attr.dtype in ['float','double']:
      x = [random.random() * window for j in range(cellcount)]
    else: # assume int or uint type
      x = [random.randint(1,window) for j in range(cellcount)]
    result.append(x)
  return result

def map_chunksize(z,minc,maxc,chunksize):
  global z_range
  maxc = 1. * maxc * chunksize
  minc = 1. * minc * chunksize
  return math.ceil(1. * z / z_range * (maxc - minc) + minc)

# create schema and partition objects
(name,attributes,dimensions) = parser.ScidbSchema.parse_schema(myschema)
Schema = parser.ScidbSchema(name,attributes,dimensions)
(range_starts,range_widths) = RangePartition.parse(partition_defs[myrange1])
range1 = RangePartition(Schema,range_starts,range_widths)
(range_starts,range_widths) = RangePartition.parse(partition_defs[myrange2])
range2 = RangePartition(Schema,range_starts,range_widths)

numnodes = max(range1.nodecount,range2.nodecount)
overlaps = []
non_overlaps1 = []
non_overlaps2 = []

#compute overlap for all nodes
for i in range(numnodes):
  #print "range1 chunks on node "+str(i)+":",range1.get_chunks(i)
  #print "range2 chunks on node "+str(i)+":",range2.get_chunks(i)
  (overlap,non_overlap1,non_overlap2) = range1.compute_overlap(range2,i)
  overlaps.append(overlap)
  if len(non_overlap1) > 0:
    non_overlaps1.append(non_overlap1)
  if len(non_overlap2) > 0:
    non_overlaps2.append(non_overlap2)
  #print "overlap for node",i,"on range1 and range2:",overlap
  #print "non-overlap for node",i,"on range1:",non_overlap1
  #print "non-overlap for node"+str(i)+" on range2:",non_overlap2

non_overlaps = None
if len(non_overlaps1) < len(non_overlaps2):
  non_overlaps = non_overlaps2
else:
  non_overlaps = non_overlaps1

total_regions = len(overlaps) + len(non_overlaps)

print total_regions,"total regions of overlap and non-overlap"

#generate the cellcounts
io.reset_offset()
io.reset_offset(aorb='B')
# prepare file handles
(dim_handlesA,attr_handlesA,chunkmap_handleA) = io.file_setup(Schema,path_prefix='data')
(dim_handlesB,attr_handlesB,chunkmap_handleB) = io.file_setup(Schema,aorb='B',path_prefix='data')
totalchunks = Schema.compute_totalchunks()
print "total chunks in",Schema.name,":",totalchunks
chunksize = Schema.compute_chunksize()
print "chunk size of",Schema.name,":",chunksize
#get zipf distribution ready
z = zipf_variable(DEFAULT_ALPHA,z_range)
#generate cellcounts for each chunk
(hotchunks,coldchunks) = get_hotcold_cellcounts(totalchunks,z_range,distribution=z)
numhot = len(hotchunks)
numhot_perregion = numhot / total_regions
hotcounts = [numhot_perregion] * total_regions

#fraction of hot chunks *not* aligned in B
numnotaligned = int(1. * numhot * join_alignment)
# divided evenly across all regions
numnotaligned_perregion = numnotaligned / total_regions
nonaligncounts = [numnotaligned_perregion] * total_regions

remainder = numhot - total_regions * numhot_perregion
remainder_nonaligned = numnotaligned - numnotaligned_perregion * total_regions
for i in range(total_regions):
  if remainder > 0:
    hotcounts[i] +=1
    remainder -= 1
  if remainder_nonaligned > 0:
    nonaligncounts[i] +=1
    remainder_nonaligned -= 1

print "total hot chunks: ",numhot
print "hot chunks per region:",hotcounts
print "total chunks *not* aligned in B:",numnotaligned
print "non-aligned chunks per region:",nonaligncounts

# put all these regions together, doesn't matter because they don't overlap
regions = overlaps + non_overlaps

hot_indexA = 0
cold_indexA = 0
hot_indexB = 0
cold_indexB = 0
cellcountsA = [0] * totalchunks # cell counts for all chunks
cellcountsB = [0] * totalchunks # cell counts for all chunks

#Build array A
#arrange cellcounts per region:
for i,region in enumerate(regions):
  #this is a list
  #pick k chunks from this region to be hot
  h = hotcounts[i]
  x = nonaligncounts[i]
  # can actually shuffle first k+x elements here, where x is # hotchunks in array B that should
  # *not* align with hot chunks in array A for this region
  k = shuffle_firstk(region,h+x) #list of chunk indexes with first k shuffled
  regions[i] = k # save this for later for use with array B
  #print "k:",k[:100]

  #first k are made hot in A
  for j in range(h):
    cellcountsA[k[j]] = hotchunks[hot_indexA] #the chunk at index k[j] is now hot
    hot_indexA += 1

  #the rest are cold in A
  for j in range(h,len(k)):
    cellcountsA[k[j]] = coldchunks[cold_indexA] #the chunk at index k[j] is now cold
    cold_indexA += 1

  for j in (range(h-x)+range(h,h+x)):
    cellcountsB[k[j]] = hotchunks[hot_indexB]
    hot_indexB += 1

  for j in (range(h-x,h)+range(h+x,len(k))):
    cellcountsB[k[j]] = coldchunks[cold_indexB]
    cold_indexB += 1


if (hot_indexA != len(hotchunks)) or (cold_indexA != len(coldchunks)):
  print "hot_indexA:",hot_indexA,",hotchunk count:",len(hotchunks)
  print "cold_indexA:",cold_indexA,",coldchunk count:",len(coldchunks)
  raise Exception("bad math here for array A")

if (hot_indexB != len(hotchunks)) or (cold_indexB != len(coldchunks)):
  print "hot_index:",hot_indexB,",hotchunk count:",len(hotchunks)
  print "cold_index:",cold_indexB,",coldchunk count:",len(coldchunks)
  raise Exception("bad math here for array B")


#print "raw cellcounts for array A:",cellcountsA,",mean:",np.mean(cellcountsA)

for i in range(1,z_range+1):
  print str(i)+":",cellcountsA.count(i)

for i in range(totalchunks):
  cellcountsA[i] = map_chunksize(cellcountsA[i],min_chunksize,max_chunksize,chunksize)
  cellcountsB[i] = map_chunksize(cellcountsB[i],min_chunksize,max_chunksize,chunksize)

print "new cellcounts for array A:",cellcountsA,",mean:",np.mean(cellcountsA)
print "regions:",regions
print "totalchunks:",totalchunks
#print "mean cellcount:",np.mean(cellcountsA)

#generate and write chunks to disk
for i in range(totalchunks):
  (chunkA,chunkB) = build_ABchunks(Schema,i,int(cellcountsA[i]),int(cellcountsB[i]),chunksize)
  #chunk = build_chunk(Schema,i,int(cellcountsA[i]),chunksize)
  #io.write_chunk_binary(chunkA,dim_handlesA,attr_handlesA,chunkmap_handleA)
  io.write_chunk(chunkA,dim_handlesA,attr_handlesA,chunkmap_handleA)
  #io.write_chunk_binary(chunkB,dim_handlesB,attr_handlesB,chunkmap_handleB)
  io.write_chunk(chunkB,dim_handlesB,attr_handlesB,chunkmap_handleB,aorb='B')

io.close_handles(dim_handlesA)
io.close_handles(attr_handlesA)
io.close_handle(chunkmap_handleA)
io.close_handles(dim_handlesB)
io.close_handles(attr_handlesB)
io.close_handle(chunkmap_handleB)

<file_sep>import schema_parser as parser
from partition import Partition #do I need this?
from range_partition import RangePartition

schema_defs = [
"array1<attr1:double,attr2:int64>[dim1=1:100,25,0,dim2=1:100,25,0]"
#,"array2<attr1:double,attr2:int64>[dim1=1:10,5,0,dim2=0:9,5,0]"
#,"array4<attr1:double,attr2:int64>[dim1=0:9,5,0,dim2=0:9,5,0]"
#,"array3<attr1:double,attr2:int64>[dim1=1:100,10,0,dim2=0:99,20,0]"
]

partition_defs = {
# quarters
'quarters':"0:1,1,50,50\
|2:1,51,50,50\
|1:51,1,50,50\
|3:51,51,50,50"
# rows
,'rows':"0:1,1,25,100\
|1:26,1,25,100\
|2:51,1,25,100\
|3:76,1,25,100"
}

(name,attributes,dimensions) = parser.ScidbSchema.parse_schema(schema_defs[0])
Schema = parser.ScidbSchema(name,attributes,dimensions)
(range_starts,range_widths) = RangePartition.parse(partition_defs['quarters'])
print "range starts for quarters:",range_starts
print "range widths for quarters:",range_widths
range1 = RangePartition(Schema,range_starts,range_widths)
(range_starts,range_widths) = RangePartition.parse(partition_defs['rows'])
print "range starts for rows:",range_starts
print "range widths for rows:",range_widths
range2 = RangePartition(Schema,range_starts,range_widths)

numnodes = max(range1.nodecount,range2.nodecount)
overlaps = []
non_overlaps1 = []
non_overlaps2 = []

#compute overlap for node 0
for i in range(numnodes):
  #print "range1 chunks on node "+str(i)+":",range1.get_chunks(i)
  #print "range2 chunks on node "+str(i)+":",range2.get_chunks(i)
  (overlap,non_overlap1,non_overlap2) = range1.compute_overlap(range2,i)
  overlaps.append(overlap)
  if len(non_overlap1) > 0:
    non_overlaps1.append(non_overlap1)
  if len(non_overlap2) > 0:
    non_overlaps2.append(non_overlap2)
  print "overlap for node",i,"on range1 and range2:",overlap
  print "non-overlap for node",i,"on range1:",non_overlap1
  print "non-overlap for node"+str(i)+" on range2:",non_overlap2

non_overlaps = None
if len(non_overlaps1) < len(non_overlaps2):
  non_overlaps = non_overlaps2
else:
  non_overlaps = non_overlaps1

total_regions = len(overlaps) + len(non_overlaps)

print total_regions,"total regions of overlap and non-overlap"


<file_sep>class ScidbDimension(object):
  '''
  Class used to store SciDB dimension information
  '''
  def __init__(self, name, dtype,range_start,range_end,chunk_width,chunk_overlap):
    self.name = name
    self.dtype = dtype
    self.range_start = -1
    self.range_end = -1
    self.chunk_width = -1
    self.chunk_overlap = 0

    if range_start is not None:
      self.range_start = int(range_start)
    if range_end is not None:
      self.range_end = int(range_end)
    if chunk_width is not None:
      self.chunk_width = int(chunk_width)
    if chunk_overlap is not None:
      self.chunk_overlap = int(chunk_overlap)

  def __repr__(self):
    s = "ScidbDimension(%s,%s,%d,%d,%d,%d)" % \
        (self.name,self.dtype,self.range_start,self.range_end,self.chunk_width,self.chunk_overlap)
    return s
<file_sep>class ScidbAttribute(object):
  '''
  Class used to store SciDB attribute information
  '''
  def __init__(self, name, dtype):
    self.name = name;
    self.dtype = dtype;

  def __repr__(self):
    s = "ScidbAttribute(%s,%s)" % (self.name,self.dtype)
    return s
<file_sep>data_generator
==============

data generator for joins project
<file_sep>class Partition(object):
  def __init__(self,Schema,range_starts,range_widths):
    self.Schema = Schema
    self.range_starts = []
    self.range_widths = []
    self.nodecount = 0

    if range_starts is not None:
      self.range_starts = range_starts
      self.nodecount = len(range_starts)
    if range_widths is not None:
      self.range_widths = range_widths

  def get_chunks(self,node_id):
    '''
    Returns the list of chunk id's associated with the given node
    using this partitioning scheme
    '''
    return []

  def compute_overlap(self,other,node_id):
    '''
    Returns the list of chunk id's that are shared between this partitioning scheme and another
    partitioning scheme for the given node
    Arguments:
      other -- an instance of the Partition class to compare with
      node_id -- the node to compare for overlap between the partitioning schemes
    '''
    return []
<file_sep>from scidb_schema import ScidbSchema
from scidb_attribute import ScidbAttribute
from scidb_dimension import ScidbDimension

print "testing SciDB schema parse functon"
schema_def = "array<attr1:double,attr2:int64>[dim1=1:10,10,0,dim2=0:9,10,0]"
(name,attributes,dimensions) = ScidbSchema.parse_schema(schema_def)

print "name:",name
print "attributes:",attributes
print "dimensions:",dimensions
