{
  "schemaVersion": 1,
  "deviceProperties": [
    {
      "id": 0, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 1, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 2, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 3, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 4, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 5, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 6, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    },
    {
      "id": 7, "name": "NVIDIA H200", "totalGlobalMem": 149631664128,
      "computeMajor": 9, "computeMinor": 0,
      "maxThreadsPerBlock": 1024, "maxThreadsPerMultiprocessor": 2048,
      "regsPerBlock": 65536, "warpSize": 32,
      "sharedMemPerBlock": 49152, "numSms": 132
    , "regsPerMultiprocessor": 65536, "sharedMemPerBlockOptin": 232448, "sharedMemPerMultiprocessor": 233472
    }
  ],
  "cupti_version": 22,
  "cuda_runtime_version": 12040,
  "cuda_driver_version": 12080,
  "distributedInfo": {"backend": "nccl", "rank": 4, "world_size": 8, "pg_count": 1, "pg_config": [{"pg_name": "0", "pg_desc": "default_pg", "backend_config": "cuda:nccl", "pg_size": 8, "ranks": [0, 1, 2, 3, 4, 5, 6, 7]}], "nccl_version": "2.21.5"},
  "record_shapes": 1,
  "trace_id": "EA67165D505F44A896A11528F951E9B3",
  "traceEvents": [
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: DivBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656702705.418, "dur": 146.894,
    "args": {
      "External id": 2483201,"Record function id": 0, "Sequence number": 24740650, "Fwd thread id": 1, "Ev Idx": 0
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "DivBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656702728.598, "dur": 109.909,
    "args": {
      "External id": 2483202,"Sequence number": 24740650, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 1
    }
  },
  {
    "ph": "f", "id": 1, "pid": 1336757, "tid": 1381170, "ts": 1514656702728.598,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336757, "tid": 1381170,
    "ts": 1514656702739.256, "dur": 96.833,
    "args": {
      "External id": 2483203,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 2
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656702867.704, "dur": 332.637,
    "args": {
      "External id": 2483204,"Record function id": 0, "Ev Idx": 3
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656702949.828, "dur": 155.690,
    "args": {
      "External id": 2483205,"Record function id": 0, "Ev Idx": 4
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.27", "pid": 1336757, "tid": 1381170,
    "ts": 1514656703033.991, "dur": 59.188,
    "args": {
      "External id": 2483206,"Record function id": 0, "Ev Idx": 5
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656703111.489, "dur": 2.174,
    "args": {
      "External id": 2483207,"Sequence number": 24740649, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 6
    }
  },
  {
    "ph": "f", "id": 2, "pid": 1336757, "tid": 1381170, "ts": 1514656703111.489,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656703119.437, "dur": 73.249,
    "args": {
      "External id": 2483208,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 7
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656703130.549, "dur": 61.564,
    "args": {
      "External id": 2483209,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 8
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656703144.455, "dur": 5.083,
    "args": {
      "External id": 2483210,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656703211.999, "dur": 35828.365,
    "args": {
      "External id": 2483211,"Record function id": 0, "Sequence number": 24740647, "Fwd thread id": 1, "Ev Idx": 10
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656703214.018, "dur": 35812.081,
    "args": {
      "External id": 2483212,"Sequence number": 24740647, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11
    }
  },
  {
    "ph": "f", "id": 3, "pid": 1336757, "tid": 1381170, "ts": 1514656703214.018,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656703269.457, "dur": 4.097,
    "args": {
      "External id": 2483213,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 12
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656703279.497, "dur": 35587.567,
    "args": {
      "External id": 2483214,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 13
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656703282.256, "dur": 35584.383,
    "args": {
      "External id": 2483215,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 14
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656703288.395, "dur": 8.459,
    "args": {
      "External id": 2483216,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 15
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656703299.231, "dur": 35565.778,
    "args": {
      "External id": 2483217,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 16
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336757, "tid": 1381170,
    "ts": 1514656738872.485, "dur": 0.547,
    "args": {
      "External id": 2483218,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 17
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656738875.879, "dur": 3.243,
    "args": {
      "External id": 2483219,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 18
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656738877.628, "dur": 1.362,
    "args": {
      "External id": 2483220,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 19
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336757, "tid": 1381170,
    "ts": 1514656738886.126, "dur": 38.298,
    "args": {
      "External id": 2483221,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 20
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336757, "tid": 1381170,
    "ts": 1514656738934.610, "dur": 79.530,
    "args": {
      "External id": 2483222,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 21
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336757, "tid": 1381170,
    "ts": 1514656738936.880, "dur": 77.049,
    "args": {
      "External id": 2483223,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 22
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336757, "tid": 1381170,
    "ts": 1514656738939.386, "dur": 73.752,
    "args": {
      "External id": 2483224,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 23
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739054.495, "dur": 24.556,
    "args": {
      "External id": 2483225,"Record function id": 0, "Sequence number": 24740646, "Fwd thread id": 1, "Ev Idx": 24
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739056.892, "dur": 18.222,
    "args": {
      "External id": 2483226,"Sequence number": 24740646, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 25
    }
  },
  {
    "ph": "f", "id": 4, "pid": 1336757, "tid": 1381170, "ts": 1514656739056.892,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739061.734, "dur": 13.171,
    "args": {
      "External id": 2483227,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 26
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739066.939, "dur": 7.793,
    "args": {
      "External id": 2483228,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 27
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739083.802, "dur": 122.533,
    "args": {
      "External id": 2483229,"Record function id": 0, "Sequence number": 24740645, "Fwd thread id": 1, "Ev Idx": 28
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739085.028, "dur": 111.028,
    "args": {
      "External id": 2483230,"Sequence number": 24740645, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 29
    }
  },
  {
    "ph": "f", "id": 5, "pid": 1336757, "tid": 1381170, "ts": 1514656739085.028,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739088.728, "dur": 106.739,
    "args": {
      "External id": 2483231,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 30
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739096.648, "dur": 41.891,
    "args": {
      "External id": 2483232,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 31
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739101.529, "dur": 7.096,
    "args": {
      "External id": 2483233,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 32
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739111.060, "dur": 27.177,
    "args": {
      "External id": 2483234,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 33
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739114.573, "dur": 23.210,
    "args": {
      "External id": 2483235,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 34
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739144.257, "dur": 4.731,
    "args": {
      "External id": 2483236,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 35
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739147.228, "dur": 1.444,
    "args": {
      "External id": 2483237,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 36
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739150.014, "dur": 44.580,
    "args": {
      "External id": 2483238,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 37
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739210.955, "dur": 77.321,
    "args": {
      "External id": 2483239,"Record function id": 0, "Sequence number": 24740644, "Fwd thread id": 1, "Ev Idx": 38
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739211.911, "dur": 70.851,
    "args": {
      "External id": 2483240,"Sequence number": 24740644, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 39
    }
  },
  {
    "ph": "f", "id": 6, "pid": 1336757, "tid": 1381170, "ts": 1514656739211.911,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739215.835, "dur": 66.695,
    "args": {
      "External id": 2483241,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "3"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 40
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739222.477, "dur": 26.703,
    "args": {
      "External id": 2483242,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 41
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739223.558, "dur": 3.770,
    "args": {
      "External id": 2483243,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 42
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739228.031, "dur": 20.853,
    "args": {
      "External id": 2483244,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 43
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739232.129, "dur": 16.351,
    "args": {
      "External id": 2483245,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 44
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739254.484, "dur": 6.235,
    "args": {
      "External id": 2483246,"Record function id": 0, "Concrete Inputs": ["", "2", "3"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 45
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739258.962, "dur": 0.984,
    "args": {
      "External id": 2483247,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 46
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739261.538, "dur": 20.340,
    "args": {
      "External id": 2483248,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 47
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739292.683, "dur": 129.841,
    "args": {
      "External id": 2483249,"Record function id": 0, "Sequence number": 24740643, "Fwd thread id": 1, "Ev Idx": 48
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739293.531, "dur": 124.728,
    "args": {
      "External id": 2483250,"Sequence number": 24740643, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 49
    }
  },
  {
    "ph": "f", "id": 7, "pid": 1336757, "tid": 1381170, "ts": 1514656739293.531,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739297.726, "dur": 120.182,
    "args": {
      "External id": 2483251,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 50
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739300.248, "dur": 16.171,
    "args": {
      "External id": 2483252,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 51
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739301.027, "dur": 2.124,
    "args": {
      "External id": 2483253,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 52
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739303.953, "dur": 12.139,
    "args": {
      "External id": 2483254,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 53
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739305.084, "dur": 10.651,
    "args": {
      "External id": 2483255,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 54
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739320.376, "dur": 2.734,
    "args": {
      "External id": 2483256,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 55
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739321.989, "dur": 0.939,
    "args": {
      "External id": 2483257,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 56
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739323.715, "dur": 93.317,
    "args": {
      "External id": 2483258,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 57
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739429.544, "dur": 115.914,
    "args": {
      "External id": 2483259,"Record function id": 0, "Sequence number": 24740642, "Fwd thread id": 1, "Ev Idx": 58
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739430.367, "dur": 111.220,
    "args": {
      "External id": 2483260,"Sequence number": 24740642, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 59
    }
  },
  {
    "ph": "f", "id": 8, "pid": 1336757, "tid": 1381170, "ts": 1514656739430.367,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739432.396, "dur": 108.878,
    "args": {
      "External id": 2483261,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 60
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739436.519, "dur": 20.482,
    "args": {
      "External id": 2483262,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 61
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739437.474, "dur": 5.097,
    "args": {
      "External id": 2483263,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 62
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739443.223, "dur": 13.515,
    "args": {
      "External id": 2483264,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 63
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739444.465, "dur": 11.863,
    "args": {
      "External id": 2483265,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 64
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739457.900, "dur": 3.952,
    "args": {
      "External id": 2483266,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 65
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739461.234, "dur": 0.451,
    "args": {
      "External id": 2483267,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 66
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739465.092, "dur": 75.159,
    "args": {
      "External id": 2483268,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 67
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739552.432, "dur": 46.664,
    "args": {
      "External id": 2483269,"Record function id": 0, "Sequence number": 24740641, "Fwd thread id": 1, "Ev Idx": 68
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739554.045, "dur": 1.435,
    "args": {
      "External id": 2483270,"Sequence number": 24740641, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 69
    }
  },
  {
    "ph": "f", "id": 9, "pid": 1336757, "tid": 1381170, "ts": 1514656739554.045,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739558.569, "dur": 35.333,
    "args": {
      "External id": 2483271,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 70
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739560.906, "dur": 32.476,
    "args": {
      "External id": 2483272,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 71
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739571.642, "dur": 0.658,
    "args": {
      "External id": 2483273,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 72
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739604.237, "dur": 2220.180,
    "args": {
      "External id": 2483274,"Record function id": 0, "Sequence number": 24740639, "Fwd thread id": 1, "Ev Idx": 73
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739605.723, "dur": 2176.039,
    "args": {
      "External id": 2483275,"Sequence number": 24740639, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 74
    }
  },
  {
    "ph": "f", "id": 10, "pid": 1336757, "tid": 1381170, "ts": 1514656739605.723,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739649.009, "dur": 2.875,
    "args": {
      "External id": 2483276,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 75
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739654.624, "dur": 2035.179,
    "args": {
      "External id": 2483277,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 76
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739656.299, "dur": 2033.235,
    "args": {
      "External id": 2483278,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 77
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739659.140, "dur": 5.413,
    "args": {
      "External id": 2483279,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 78
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656739665.520, "dur": 2023.081,
    "args": {
      "External id": 2483280,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 79
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741693.160, "dur": 0.353,
    "args": {
      "External id": 2483281,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 80
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741697.820, "dur": 3.110,
    "args": {
      "External id": 2483282,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 81
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741699.763, "dur": 0.957,
    "args": {
      "External id": 2483283,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 82
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741705.346, "dur": 22.090,
    "args": {
      "External id": 2483284,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 83
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741733.667, "dur": 40.225,
    "args": {
      "External id": 2483285,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 84
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741734.898, "dur": 38.823,
    "args": {
      "External id": 2483286,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 85
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741735.817, "dur": 37.633,
    "args": {
      "External id": 2483287,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 86
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741795.092, "dur": 24.731,
    "args": {
      "External id": 2483288,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 87
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741831.890, "dur": 18.997,
    "args": {
      "External id": 2483289,"Record function id": 0, "Sequence number": 24740638, "Fwd thread id": 1, "Ev Idx": 88
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741833.515, "dur": 13.317,
    "args": {
      "External id": 2483290,"Sequence number": 24740638, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 89
    }
  },
  {
    "ph": "f", "id": 11, "pid": 1336757, "tid": 1381170, "ts": 1514656741833.515,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741841.066, "dur": 5.555,
    "args": {
      "External id": 2483291,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 90
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741842.470, "dur": 4.015,
    "args": {
      "External id": 2483292,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 91
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741854.438, "dur": 75.024,
    "args": {
      "External id": 2483293,"Record function id": 0, "Sequence number": 24740637, "Fwd thread id": 1, "Ev Idx": 92
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741855.673, "dur": 67.533,
    "args": {
      "External id": 2483294,"Sequence number": 24740637, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 93
    }
  },
  {
    "ph": "f", "id": 12, "pid": 1336757, "tid": 1381170, "ts": 1514656741855.673,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741857.934, "dur": 64.980,
    "args": {
      "External id": 2483295,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 94
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741862.505, "dur": 23.226,
    "args": {
      "External id": 2483296,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 95
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741864.121, "dur": 3.031,
    "args": {
      "External id": 2483297,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 96
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741868.012, "dur": 17.452,
    "args": {
      "External id": 2483298,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 97
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741869.351, "dur": 15.682,
    "args": {
      "External id": 2483299,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 98
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741887.024, "dur": 5.030,
    "args": {
      "External id": 2483300,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 99
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741888.717, "dur": 3.029,
    "args": {
      "External id": 2483301,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741892.703, "dur": 29.291,
    "args": {
      "External id": 2483302,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741935.974, "dur": 95.900,
    "args": {
      "External id": 2483303,"Record function id": 0, "Sequence number": 24740636, "Fwd thread id": 1, "Ev Idx": 102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741936.888, "dur": 90.145,
    "args": {
      "External id": 2483304,"Sequence number": 24740636, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 103
    }
  },
  {
    "ph": "f", "id": 13, "pid": 1336757, "tid": 1381170, "ts": 1514656741936.888,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741941.018, "dur": 85.746,
    "args": {
      "External id": 2483305,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "2"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741942.527, "dur": 21.838,
    "args": {
      "External id": 2483306,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741943.639, "dur": 2.188,
    "args": {
      "External id": 2483307,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741946.572, "dur": 17.504,
    "args": {
      "External id": 2483308,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741947.731, "dur": 15.939,
    "args": {
      "External id": 2483309,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741965.476, "dur": 6.924,
    "args": {
      "External id": 2483310,"Record function id": 0, "Concrete Inputs": ["", "2", "2"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741970.696, "dur": 1.178,
    "args": {
      "External id": 2483311,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656741973.263, "dur": 52.534,
    "args": {
      "External id": 2483312,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742037.728, "dur": 97.704,
    "args": {
      "External id": 2483313,"Record function id": 0, "Sequence number": 24740635, "Fwd thread id": 1, "Ev Idx": 112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742041.376, "dur": 89.857,
    "args": {
      "External id": 2483314,"Sequence number": 24740635, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 113
    }
  },
  {
    "ph": "f", "id": 14, "pid": 1336757, "tid": 1381170, "ts": 1514656742041.376,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742043.824, "dur": 87.100,
    "args": {
      "External id": 2483315,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742045.127, "dur": 20.297,
    "args": {
      "External id": 2483316,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742046.093, "dur": 2.398,
    "args": {
      "External id": 2483317,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742049.190, "dur": 15.960,
    "args": {
      "External id": 2483318,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742050.326, "dur": 14.409,
    "args": {
      "External id": 2483319,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742066.599, "dur": 4.408,
    "args": {
      "External id": 2483320,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742070.082, "dur": 0.647,
    "args": {
      "External id": 2483321,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742071.617, "dur": 58.705,
    "args": {
      "External id": 2483322,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742139.585, "dur": 96.629,
    "args": {
      "External id": 2483323,"Record function id": 0, "Sequence number": 24740634, "Fwd thread id": 1, "Ev Idx": 122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742142.574, "dur": 74.255,
    "args": {
      "External id": 2483324,"Sequence number": 24740634, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 123
    }
  },
  {
    "ph": "f", "id": 15, "pid": 1336757, "tid": 1381170, "ts": 1514656742142.574,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742146.117, "dur": 70.432,
    "args": {
      "External id": 2483325,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742147.066, "dur": 17.458,
    "args": {
      "External id": 2483326,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742147.868, "dur": 1.717,
    "args": {
      "External id": 2483327,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742150.350, "dur": 13.917,
    "args": {
      "External id": 2483328,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742151.212, "dur": 12.758,
    "args": {
      "External id": 2483329,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742165.402, "dur": 1.946,
    "args": {
      "External id": 2483330,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742166.288, "dur": 0.880,
    "args": {
      "External id": 2483331,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742168.168, "dur": 47.813,
    "args": {
      "External id": 2483332,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742220.375, "dur": 14.685,
    "args": {
      "External id": 2483333,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742240.483, "dur": 41.845,
    "args": {
      "External id": 2483334,"Record function id": 0, "Sequence number": 24740633, "Fwd thread id": 1, "Ev Idx": 133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742244.553, "dur": 1.187,
    "args": {
      "External id": 2483335,"Sequence number": 24740633, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 134
    }
  },
  {
    "ph": "f", "id": 16, "pid": 1336757, "tid": 1381170, "ts": 1514656742244.553,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742248.242, "dur": 28.843,
    "args": {
      "External id": 2483336,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742250.006, "dur": 26.586,
    "args": {
      "External id": 2483337,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742255.960, "dur": 3.227,
    "args": {
      "External id": 2483338,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742287.112, "dur": 3040.980,
    "args": {
      "External id": 2483339,"Record function id": 0, "Sequence number": 24740631, "Fwd thread id": 1, "Ev Idx": 138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742290.937, "dur": 3007.020,
    "args": {
      "External id": 2483340,"Sequence number": 24740631, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 139
    }
  },
  {
    "ph": "f", "id": 17, "pid": 1336757, "tid": 1381170, "ts": 1514656742290.937,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742323.412, "dur": 3.233,
    "args": {
      "External id": 2483341,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742328.576, "dur": 2874.103,
    "args": {
      "External id": 2483342,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742330.344, "dur": 2871.933,
    "args": {
      "External id": 2483343,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742332.750, "dur": 2.830,
    "args": {
      "External id": 2483344,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656742336.451, "dur": 2864.328,
    "args": {
      "External id": 2483345,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745207.077, "dur": 0.378,
    "args": {
      "External id": 2483346,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745208.985, "dur": 2.583,
    "args": {
      "External id": 2483347,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745210.559, "dur": 0.875,
    "args": {
      "External id": 2483348,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745216.504, "dur": 23.243,
    "args": {
      "External id": 2483349,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745250.215, "dur": 39.917,
    "args": {
      "External id": 2483350,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745251.517, "dur": 38.361,
    "args": {
      "External id": 2483351,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745252.532, "dur": 36.940,
    "args": {
      "External id": 2483352,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745308.575, "dur": 15.634,
    "args": {
      "External id": 2483353,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745337.700, "dur": 12.088,
    "args": {
      "External id": 2483354,"Record function id": 0, "Sequence number": 24740630, "Fwd thread id": 1, "Ev Idx": 153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745339.461, "dur": 6.980,
    "args": {
      "External id": 2483355,"Sequence number": 24740630, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 154
    }
  },
  {
    "ph": "f", "id": 18, "pid": 1336757, "tid": 1381170, "ts": 1514656745339.461,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745341.865, "dur": 4.366,
    "args": {
      "External id": 2483356,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745343.128, "dur": 2.980,
    "args": {
      "External id": 2483357,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745353.418, "dur": 70.069,
    "args": {
      "External id": 2483358,"Record function id": 0, "Sequence number": 24740629, "Fwd thread id": 1, "Ev Idx": 157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745354.426, "dur": 62.485,
    "args": {
      "External id": 2483359,"Sequence number": 24740629, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 158
    }
  },
  {
    "ph": "f", "id": 19, "pid": 1336757, "tid": 1381170, "ts": 1514656745354.426,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745358.715, "dur": 57.851,
    "args": {
      "External id": 2483360,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745363.723, "dur": 20.259,
    "args": {
      "External id": 2483361,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745365.100, "dur": 2.732,
    "args": {
      "External id": 2483362,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745368.658, "dur": 15.045,
    "args": {
      "External id": 2483363,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745370.160, "dur": 13.136,
    "args": {
      "External id": 2483364,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745385.277, "dur": 5.141,
    "args": {
      "External id": 2483365,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745389.179, "dur": 1.015,
    "args": {
      "External id": 2483366,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745391.215, "dur": 24.556,
    "args": {
      "External id": 2483367,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745427.696, "dur": 55.688,
    "args": {
      "External id": 2483368,"Record function id": 0, "Sequence number": 24740628, "Fwd thread id": 1, "Ev Idx": 167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745428.915, "dur": 50.960,
    "args": {
      "External id": 2483369,"Sequence number": 24740628, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 168
    }
  },
  {
    "ph": "f", "id": 20, "pid": 1336757, "tid": 1381170, "ts": 1514656745428.915,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745430.793, "dur": 48.789,
    "args": {
      "External id": 2483370,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745432.677, "dur": 20.097,
    "args": {
      "External id": 2483371,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745436.043, "dur": 2.166,
    "args": {
      "External id": 2483372,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745438.736, "dur": 13.780,
    "args": {
      "External id": 2483373,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745439.788, "dur": 12.406,
    "args": {
      "External id": 2483374,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745454.054, "dur": 9.668,
    "args": {
      "External id": 2483375,"Record function id": 0, "Concrete Inputs": ["", "2", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745460.261, "dur": 2.936,
    "args": {
      "External id": 2483376,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745464.335, "dur": 14.750,
    "args": {
      "External id": 2483377,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745487.099, "dur": 109.218,
    "args": {
      "External id": 2483378,"Record function id": 0, "Sequence number": 24740627, "Fwd thread id": 1, "Ev Idx": 177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745488.205, "dur": 103.317,
    "args": {
      "External id": 2483379,"Sequence number": 24740627, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 178
    }
  },
  {
    "ph": "f", "id": 21, "pid": 1336757, "tid": 1381170, "ts": 1514656745488.205,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745490.238, "dur": 100.999,
    "args": {
      "External id": 2483380,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745491.195, "dur": 21.887,
    "args": {
      "External id": 2483381,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745492.251, "dur": 2.159,
    "args": {
      "External id": 2483382,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745497.494, "dur": 15.312,
    "args": {
      "External id": 2483383,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745498.266, "dur": 14.127,
    "args": {
      "External id": 2483384,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745514.465, "dur": 4.319,
    "args": {
      "External id": 2483385,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745518.121, "dur": 0.503,
    "args": {
      "External id": 2483386,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745519.587, "dur": 70.496,
    "args": {
      "External id": 2483387,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745602.515, "dur": 100.462,
    "args": {
      "External id": 2483388,"Record function id": 0, "Sequence number": 24740626, "Fwd thread id": 1, "Ev Idx": 187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745603.939, "dur": 81.146,
    "args": {
      "External id": 2483389,"Sequence number": 24740626, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 188
    }
  },
  {
    "ph": "f", "id": 22, "pid": 1336757, "tid": 1381170, "ts": 1514656745603.939,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745608.690, "dur": 76.143,
    "args": {
      "External id": 2483390,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745609.781, "dur": 19.672,
    "args": {
      "External id": 2483391,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745611.097, "dur": 2.403,
    "args": {
      "External id": 2483392,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745614.306, "dur": 14.864,
    "args": {
      "External id": 2483393,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745617.832, "dur": 10.962,
    "args": {
      "External id": 2483394,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745630.553, "dur": 4.762,
    "args": {
      "External id": 2483395,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745634.323, "dur": 0.790,
    "args": {
      "External id": 2483396,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745635.801, "dur": 48.411,
    "args": {
      "External id": 2483397,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745688.767, "dur": 13.091,
    "args": {
      "External id": 2483398,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745707.387, "dur": 34.162,
    "args": {
      "External id": 2483399,"Record function id": 0, "Sequence number": 24740625, "Fwd thread id": 1, "Ev Idx": 198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745708.587, "dur": 1.292,
    "args": {
      "External id": 2483400,"Sequence number": 24740625, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 199
    }
  },
  {
    "ph": "f", "id": 23, "pid": 1336757, "tid": 1381170, "ts": 1514656745708.587,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745711.910, "dur": 23.972,
    "args": {
      "External id": 2483401,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745713.589, "dur": 21.813,
    "args": {
      "External id": 2483402,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[1], [], [], []], "Ev Idx": 201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745718.653, "dur": 0.499,
    "args": {
      "External id": 2483403,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FusedLinearCrossEntropyFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745746.156, "dur": 3073.790,
    "args": {
      "External id": 2483404,"Record function id": 0, "Sequence number": 24740624, "Fwd thread id": 1, "Ev Idx": 203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745760.272, "dur": 3027.387,
    "args": {
      "External id": 2483405,"Sequence number": 24740624, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 204
    }
  },
  {
    "ph": "f", "id": 24, "pid": 1336757, "tid": 1381170, "ts": 1514656745760.272,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745789.086, "dur": 2.446,
    "args": {
      "External id": 2483406,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745793.836, "dur": 2911.483,
    "args": {
      "External id": 2483407,"Record function id": 0, "Concrete Inputs": ["", "", "6", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745795.013, "dur": 2910.057,
    "args": {
      "External id": 2483408,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745800.082, "dur": 2.693,
    "args": {
      "External id": 2483409,"Record function id": 0, "Concrete Inputs": ["[]", "[]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656745803.433, "dur": 2900.879,
    "args": {
      "External id": 2483410,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::lift_fresh", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748708.477, "dur": 0.469,
    "args": {
      "External id": 2483411,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748710.113, "dur": 1.916,
    "args": {
      "External id": 2483412,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748711.255, "dur": 0.648,
    "args": {
      "External id": 2483413,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748715.663, "dur": 22.835,
    "args": {
      "External id": 2483414,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748743.363, "dur": 37.686,
    "args": {
      "External id": 2483415,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748744.343, "dur": 36.526,
    "args": {
      "External id": 2483416,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748745.637, "dur": 34.873,
    "args": {
      "External id": 2483417,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748798.805, "dur": 15.605,
    "args": {
      "External id": 2483418,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748834.629, "dur": 14.649,
    "args": {
      "External id": 2483419,"Record function id": 0, "Ev Idx": 218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748837.542, "dur": 9.648,
    "args": {
      "External id": 2483420,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748841.356, "dur": 4.758,
    "args": {
      "External id": 2483421,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748842.459, "dur": 3.513,
    "args": {
      "External id": 2483422,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748852.709, "dur": 12.134,
    "args": {
      "External id": 2483423,"Record function id": 0, "Sequence number": 24740623, "Fwd thread id": 1, "Ev Idx": 222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748854.002, "dur": 8.404,
    "args": {
      "External id": 2483424,"Sequence number": 24740623, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 223
    }
  },
  {
    "ph": "f", "id": 25, "pid": 1336757, "tid": 1381170, "ts": 1514656748854.002,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748856.158, "dur": 6.056,
    "args": {
      "External id": 2483425,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748859.539, "dur": 2.559,
    "args": {
      "External id": 2483426,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748868.338, "dur": 75.197,
    "args": {
      "External id": 2483427,"Record function id": 0, "Sequence number": 24740622, "Fwd thread id": 1, "Ev Idx": 226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748869.537, "dur": 68.146,
    "args": {
      "External id": 2483428,"Sequence number": 24740622, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 227
    }
  },
  {
    "ph": "f", "id": 26, "pid": 1336757, "tid": 1381170, "ts": 1514656748869.537,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748871.135, "dur": 66.259,
    "args": {
      "External id": 2483429,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748876.404, "dur": 23.096,
    "args": {
      "External id": 2483430,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748877.718, "dur": 2.409,
    "args": {
      "External id": 2483431,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748881.144, "dur": 18.045,
    "args": {
      "External id": 2483432,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748882.374, "dur": 16.456,
    "args": {
      "External id": 2483433,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748900.880, "dur": 5.033,
    "args": {
      "External id": 2483434,"Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748904.928, "dur": 0.794,
    "args": {
      "External id": 2483435,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[8388608, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748906.941, "dur": 29.806,
    "args": {
      "External id": 2483436,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SelectBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748949.813, "dur": 85.227,
    "args": {
      "External id": 2483437,"Record function id": 0, "Sequence number": 24740621, "Fwd thread id": 1, "Ev Idx": 236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SelectBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748950.899, "dur": 80.580,
    "args": {
      "External id": 2483438,"Sequence number": 24740621, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 237
    }
  },
  {
    "ph": "f", "id": 27, "pid": 1336757, "tid": 1381170, "ts": 1514656748950.899,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748953.244, "dur": 77.980,
    "args": {
      "External id": 2483439,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "2", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748954.833, "dur": 24.485,
    "args": {
      "External id": 2483440,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748962.139, "dur": 2.223,
    "args": {
      "External id": 2483441,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748965.041, "dur": 14.017,
    "args": {
      "External id": 2483442,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748966.135, "dur": 12.541,
    "args": {
      "External id": 2483443,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1381170,
    "ts": 1514656748980.436, "dur": 30.104,
    "args": {
      "External id": 2483444,"Record function id": 0, "Concrete Inputs": ["", "2", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749008.694, "dur": 1.033,
    "args": {
      "External id": 2483445,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749011.651, "dur": 18.898,
    "args": {
      "External id": 2483446,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749041.026, "dur": 94.040,
    "args": {
      "External id": 2483447,"Record function id": 0, "Sequence number": 24740620, "Fwd thread id": 1, "Ev Idx": 246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749042.071, "dur": 89.202,
    "args": {
      "External id": 2483448,"Sequence number": 24740620, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 247
    }
  },
  {
    "ph": "f", "id": 28, "pid": 1336757, "tid": 1381170, "ts": 1514656749042.071,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749044.186, "dur": 86.770,
    "args": {
      "External id": 2483449,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749047.680, "dur": 20.663,
    "args": {
      "External id": 2483450,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749048.761, "dur": 2.531,
    "args": {
      "External id": 2483451,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749054.036, "dur": 14.053,
    "args": {
      "External id": 2483452,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749055.334, "dur": 12.401,
    "args": {
      "External id": 2483453,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749069.277, "dur": 4.468,
    "args": {
      "External id": 2483454,"Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749070.821, "dur": 2.713,
    "args": {
      "External id": 2483455,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749074.305, "dur": 55.958,
    "args": {
      "External id": 2483456,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749138.913, "dur": 105.907,
    "args": {
      "External id": 2483457,"Record function id": 0, "Sequence number": 24740619, "Fwd thread id": 1, "Ev Idx": 256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SliceBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749140.042, "dur": 84.442,
    "args": {
      "External id": 2483458,"Sequence number": 24740619, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 257
    }
  },
  {
    "ph": "f", "id": 29, "pid": 1336757, "tid": 1381170, "ts": 1514656749140.042,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749141.288, "dur": 82.795,
    "args": {
      "External id": 2483459,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], [], []], "Ev Idx": 258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749142.319, "dur": 30.935,
    "args": {
      "External id": 2483460,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749147.955, "dur": 1.799,
    "args": {
      "External id": 2483461,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 4, 2048]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749154.217, "dur": 18.762,
    "args": {
      "External id": 2483462,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749159.768, "dur": 12.875,
    "args": {
      "External id": 2483463,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], []], "Ev Idx": 262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749174.187, "dur": 1.780,
    "args": {
      "External id": 2483464,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749175.264, "dur": 0.508,
    "args": {
      "External id": 2483465,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749176.595, "dur": 46.936,
    "args": {
      "External id": 2483466,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749228.452, "dur": 14.259,
    "args": {
      "External id": 2483467,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [33554432, 8192, 2048, 1], []], "Input Dims": [[16, 4096, 4, 2048], [16, 4096, 4, 2048], []], "Ev Idx": 266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749250.719, "dur": 429.181,
    "args": {
      "External id": 2483468,"Record function id": 0, "Sequence number": 24740618, "Fwd thread id": 1, "Ev Idx": 267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749252.057, "dur": 415.940,
    "args": {
      "External id": 2483469,"Sequence number": 24740618, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 268
    }
  },
  {
    "ph": "f", "id": 30, "pid": 1336757, "tid": 1381170, "ts": 1514656749252.057,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749459.381, "dur": 50.309,
    "args": {
      "External id": 2483470,"kernel_hash": "cl7ukpd4jntugtfn4xeypgd5onvbe6twamcfekvhw4dgeizu3kp2", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "262144", "2048", "1", "1986", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/l7/cl7ukpd4jntugtfn4xeypgd5onvbe6twamcfekvhw4dgeizu3kp2.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[262144, 2048], [2048], [262144, 2048], [262144, 2048], [132, 2048], [262144], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749565.289, "dur": 33.563,
    "args": {
      "External id": 2483471,"kernel_hash": "ctrxskj43j6wttul7meux4tf5uhrizad56rruyqg3gopczymn3ux", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/tr/ctrxskj43j6wttul7meux4tf5uhrizad56rruyqg3gopczymn3ux.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749618.172, "dur": 20.029,
    "args": {
      "External id": 2483472,"kernel_hash": "chk6naviqoas7yeoerriva67j5bcolw65yisn4ks734r66m7rt3g", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/hk/chk6naviqoas7yeoerriva67j5bcolw65yisn4ks734r66m7rt3g.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749690.709, "dur": 14.606,
    "args": {
      "External id": 2483473,"Record function id": 0, "Ev Idx": 272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749693.255, "dur": 11.232,
    "args": {
      "External id": 2483474,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749696.368, "dur": 7.153,
    "args": {
      "External id": 2483475,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749700.141, "dur": 3.233,
    "args": {
      "External id": 2483476,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: StackBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749711.814, "dur": 42.477,
    "args": {
      "External id": 2483477,"Record function id": 0, "Sequence number": 24740617, "Fwd thread id": 1, "Ev Idx": 276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "StackBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749713.273, "dur": 27.780,
    "args": {
      "External id": 2483478,"Sequence number": 24740617, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 2048, 1]], "Input Dims": [[16, 4096, 4, 2048]], "Ev Idx": 277
    }
  },
  {
    "ph": "f", "id": 31, "pid": 1336757, "tid": 1381170, "ts": 1514656749713.273,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749715.791, "dur": 8.221,
    "args": {
      "External id": 2483479,"Record function id": 0, "Concrete Inputs": ["", "-2", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749720.953, "dur": 1.320,
    "args": {
      "External id": 2483480,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749724.680, "dur": 7.584,
    "args": {
      "External id": 2483481,"Record function id": 0, "Concrete Inputs": ["", "-2", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749730.966, "dur": 0.560,
    "args": {
      "External id": 2483482,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749733.120, "dur": 2.491,
    "args": {
      "External id": 2483483,"Record function id": 0, "Concrete Inputs": ["", "-2", "2"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749734.229, "dur": 0.678,
    "args": {
      "External id": 2483484,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749736.184, "dur": 4.264,
    "args": {
      "External id": 2483485,"Record function id": 0, "Concrete Inputs": ["", "-2", "3"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749739.084, "dur": 0.435,
    "args": {
      "External id": 2483486,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749758.012, "dur": 5.722,
    "args": {
      "External id": 2483487,"Record function id": 0, "Sequence number": 24740616, "Fwd thread id": 1, "Ev Idx": 286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749758.931, "dur": 1.097,
    "args": {
      "External id": 2483488,"Sequence number": 24740616, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 287
    }
  },
  {
    "ph": "f", "id": 32, "pid": 1336757, "tid": 1381170, "ts": 1514656749758.931,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749768.426, "dur": 638.573,
    "args": {
      "External id": 2483489,"Record function id": 0, "Sequence number": 24740615, "Fwd thread id": 1, "Ev Idx": 288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749769.811, "dur": 619.464,
    "args": {
      "External id": 2483490,"Sequence number": 24740615, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 289
    }
  },
  {
    "ph": "f", "id": 33, "pid": 1336757, "tid": 1381170, "ts": 1514656749769.811,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749820.839, "dur": 14.247,
    "args": {
      "External id": 2483491,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749829.400, "dur": 5.394,
    "args": {
      "External id": 2483492,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749842.838, "dur": 7.652,
    "args": {
      "External id": 2483493,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749847.003, "dur": 2.683,
    "args": {
      "External id": 2483494,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749848.837, "dur": 0.572,
    "args": {
      "External id": 2483495,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749858.260, "dur": 180.534,
    "args": {
      "External id": 2483496,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749859.859, "dur": 7.455,
    "args": {
      "External id": 2483497,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749860.482, "dur": 6.125,
    "args": {
      "External id": 2483498,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749863.688, "dur": 2.822,
    "args": {
      "External id": 2483499,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749869.833, "dur": 168.006,
    "args": {
      "External id": 2483500,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656749872.776, "dur": 163.137,
    "args": {
      "External id": 2483501,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750047.297, "dur": 7.814,
    "args": {
      "External id": 2483502,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750051.955, "dur": 3.055,
    "args": {
      "External id": 2483503,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750100.384, "dur": 5.404,
    "args": {
      "External id": 2483504,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750106.902, "dur": 3.847,
    "args": {
      "External id": 2483505,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750113.587, "dur": 2.224,
    "args": {
      "External id": 2483506,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750157.334, "dur": 1.987,
    "args": {
      "External id": 2483507,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750158.005, "dur": 1.155,
    "args": {
      "External id": 2483508,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750194.022, "dur": 172.191,
    "args": {
      "External id": 2483509,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750201.521, "dur": 10.070,
    "args": {
      "External id": 2483510,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750208.158, "dur": 1.014,
    "args": {
      "External id": 2483511,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750214.009, "dur": 10.705,
    "args": {
      "External id": 2483512,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750220.429, "dur": 3.227,
    "args": {
      "External id": 2483513,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750226.956, "dur": 3.730,
    "args": {
      "External id": 2483514,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750229.866, "dur": 0.455,
    "args": {
      "External id": 2483515,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750231.455, "dur": 4.446,
    "args": {
      "External id": 2483516,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750234.671, "dur": 0.593,
    "args": {
      "External id": 2483517,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750241.980, "dur": 4.687,
    "args": {
      "External id": 2483518,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750245.878, "dur": 0.489,
    "args": {
      "External id": 2483519,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750249.248, "dur": 6.979,
    "args": {
      "External id": 2483520,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750254.116, "dur": 1.929,
    "args": {
      "External id": 2483521,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750256.995, "dur": 4.437,
    "args": {
      "External id": 2483522,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750260.647, "dur": 0.511,
    "args": {
      "External id": 2483523,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750262.651, "dur": 2.576,
    "args": {
      "External id": 2483524,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750263.605, "dur": 1.526,
    "args": {
      "External id": 2483525,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750267.299, "dur": 83.039,
    "args": {
      "External id": 2483526,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750352.426, "dur": 3.477,
    "args": {
      "External id": 2483527,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750359.382, "dur": 2.422,
    "args": {
      "External id": 2483528,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750360.646, "dur": 0.527,
    "args": {
      "External id": 2483529,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750363.901, "dur": 1.021,
    "args": {
      "External id": 2483530,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750419.724, "dur": 10.708,
    "args": {
      "External id": 2483531,"Record function id": 0, "Ev Idx": 330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750421.821, "dur": 7.926,
    "args": {
      "External id": 2483532,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750424.649, "dur": 4.215,
    "args": {
      "External id": 2483533,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750426.016, "dur": 2.702,
    "args": {
      "External id": 2483534,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750434.778, "dur": 10.763,
    "args": {
      "External id": 2483535,"Record function id": 0, "Sequence number": 24740614, "Fwd thread id": 1, "Ev Idx": 334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750435.896, "dur": 6.302,
    "args": {
      "External id": 2483536,"Sequence number": 24740614, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 335
    }
  },
  {
    "ph": "f", "id": 34, "pid": 1336757, "tid": 1381170, "ts": 1514656750435.896,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750440.177, "dur": 1.815,
    "args": {
      "External id": 2483537,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750440.970, "dur": 0.914,
    "args": {
      "External id": 2483538,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750450.155, "dur": 183.337,
    "args": {
      "External id": 2483539,"Record function id": 0, "Sequence number": 24740613, "Fwd thread id": 1, "Ev Idx": 338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750451.132, "dur": 172.516,
    "args": {
      "External id": 2483540,"Sequence number": 24740613, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 339
    }
  },
  {
    "ph": "f", "id": 35, "pid": 1336757, "tid": 1381170, "ts": 1514656750451.132,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750458.662, "dur": 4.422,
    "args": {
      "External id": 2483541,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750460.060, "dur": 2.463,
    "args": {
      "External id": 2483542,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750461.705, "dur": 0.623,
    "args": {
      "External id": 2483543,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750464.747, "dur": 55.428,
    "args": {
      "External id": 2483544,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750521.489, "dur": 23.556,
    "args": {
      "External id": 2483545,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750522.518, "dur": 21.493,
    "args": {
      "External id": 2483546,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750526.659, "dur": 16.805,
    "args": {
      "External id": 2483547,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750548.296, "dur": 5.831,
    "args": {
      "External id": 2483548,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750549.880, "dur": 3.784,
    "args": {
      "External id": 2483549,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750553.103, "dur": 0.484,
    "args": {
      "External id": 2483550,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750554.879, "dur": 67.908,
    "args": {
      "External id": 2483551,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750640.202, "dur": 7.985,
    "args": {
      "External id": 2483552,"Record function id": 0, "Sequence number": 24740612, "Fwd thread id": 1, "Ev Idx": 351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750641.701, "dur": 4.424,
    "args": {
      "External id": 2483553,"Sequence number": 24740612, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 352
    }
  },
  {
    "ph": "f", "id": 36, "pid": 1336757, "tid": 1381170, "ts": 1514656750641.701,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750643.554, "dur": 2.424,
    "args": {
      "External id": 2483554,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750644.445, "dur": 1.439,
    "args": {
      "External id": 2483555,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750652.043, "dur": 14.301,
    "args": {
      "External id": 2483556,"Record function id": 0, "Sequence number": 24740611, "Fwd thread id": 1, "Ev Idx": 355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750655.138, "dur": 7.153,
    "args": {
      "External id": 2483557,"Sequence number": 24740611, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 356
    }
  },
  {
    "ph": "f", "id": 37, "pid": 1336757, "tid": 1381170, "ts": 1514656750655.138,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750656.749, "dur": 5.342,
    "args": {
      "External id": 2483558,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750657.409, "dur": 4.158,
    "args": {
      "External id": 2483559,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750660.936, "dur": 0.549,
    "args": {
      "External id": 2483560,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750670.742, "dur": 6.244,
    "args": {
      "External id": 2483561,"Record function id": 0, "Ev Idx": 360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750672.113, "dur": 4.263,
    "args": {
      "External id": 2483562,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750673.582, "dur": 2.427,
    "args": {
      "External id": 2483563,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750674.231, "dur": 1.691,
    "args": {
      "External id": 2483564,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750680.308, "dur": 10.474,
    "args": {
      "External id": 2483565,"Record function id": 0, "Sequence number": 24740610, "Fwd thread id": 1, "Ev Idx": 364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750681.192, "dur": 5.154,
    "args": {
      "External id": 2483566,"Sequence number": 24740610, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 365
    }
  },
  {
    "ph": "f", "id": 38, "pid": 1336757, "tid": 1381170, "ts": 1514656750681.192,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750682.670, "dur": 3.542,
    "args": {
      "External id": 2483567,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750685.260, "dur": 0.812,
    "args": {
      "External id": 2483568,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750694.143, "dur": 127.667,
    "args": {
      "External id": 2483569,"Record function id": 0, "Sequence number": 24740609, "Fwd thread id": 1, "Ev Idx": 368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750695.067, "dur": 114.327,
    "args": {
      "External id": 2483570,"Sequence number": 24740609, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 369
    }
  },
  {
    "ph": "f", "id": 39, "pid": 1336757, "tid": 1381170, "ts": 1514656750695.067,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750698.972, "dur": 2.683,
    "args": {
      "External id": 2483571,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750699.635, "dur": 1.613,
    "args": {
      "External id": 2483572,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750700.605, "dur": 0.542,
    "args": {
      "External id": 2483573,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750704.465, "dur": 37.658,
    "args": {
      "External id": 2483574,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750743.397, "dur": 5.943,
    "args": {
      "External id": 2483575,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750744.102, "dur": 4.744,
    "args": {
      "External id": 2483576,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750747.454, "dur": 1.286,
    "args": {
      "External id": 2483577,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750750.673, "dur": 7.546,
    "args": {
      "External id": 2483578,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750751.714, "dur": 5.966,
    "args": {
      "External id": 2483579,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750754.988, "dur": 2.619,
    "args": {
      "External id": 2483580,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750758.982, "dur": 49.591,
    "args": {
      "External id": 2483581,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750828.420, "dur": 37.210,
    "args": {
      "External id": 2483582,"Record function id": 0, "Sequence number": 24740608, "Fwd thread id": 1, "Ev Idx": 381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750829.685, "dur": 5.910,
    "args": {
      "External id": 2483583,"Sequence number": 24740608, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 382
    }
  },
  {
    "ph": "f", "id": 40, "pid": 1336757, "tid": 1381170, "ts": 1514656750829.685,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750831.133, "dur": 4.308,
    "args": {
      "External id": 2483584,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750834.058, "dur": 1.259,
    "args": {
      "External id": 2483585,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750839.223, "dur": 23.153,
    "args": {
      "External id": 2483586,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750869.698, "dur": 12.621,
    "args": {
      "External id": 2483587,"Record function id": 0, "Sequence number": 24740607, "Fwd thread id": 1, "Ev Idx": 386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750871.072, "dur": 6.848,
    "args": {
      "External id": 2483588,"Sequence number": 24740607, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 387
    }
  },
  {
    "ph": "f", "id": 41, "pid": 1336757, "tid": 1381170, "ts": 1514656750871.072,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750872.203, "dur": 5.525,
    "args": {
      "External id": 2483589,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750875.505, "dur": 1.690,
    "args": {
      "External id": 2483590,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750876.574, "dur": 0.487,
    "args": {
      "External id": 2483591,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750886.374, "dur": 4.766,
    "args": {
      "External id": 2483592,"Record function id": 0, "Ev Idx": 391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750887.745, "dur": 2.896,
    "args": {
      "External id": 2483593,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750888.599, "dur": 1.623,
    "args": {
      "External id": 2483594,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750889.159, "dur": 0.983,
    "args": {
      "External id": 2483595,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750896.210, "dur": 541.273,
    "args": {
      "External id": 2483596,"Record function id": 0, "Sequence number": 24740606, "Fwd thread id": 1, "Ev Idx": 395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750897.801, "dur": 519.198,
    "args": {
      "External id": 2483597,"Sequence number": 24740606, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 396
    }
  },
  {
    "ph": "f", "id": 42, "pid": 1336757, "tid": 1381170, "ts": 1514656750897.801,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750927.680, "dur": 37.124,
    "args": {
      "External id": 2483598,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750929.856, "dur": 34.763,
    "args": {
      "External id": 2483599,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750933.070, "dur": 5.834,
    "args": {
      "External id": 2483600,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750935.779, "dur": 2.593,
    "args": {
      "External id": 2483601,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750940.656, "dur": 23.353,
    "args": {
      "External id": 2483602,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656750977.577, "dur": 37.648,
    "args": {
      "External id": 2483603,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751012.404, "dur": 2.482,
    "args": {
      "External id": 2483604,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751021.523, "dur": 3.748,
    "args": {
      "External id": 2483605,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751022.356, "dur": 2.808,
    "args": {
      "External id": 2483606,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751044.365, "dur": 3.517,
    "args": {
      "External id": 2483607,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751064.136, "dur": 3.562,
    "args": {
      "External id": 2483608,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751295.296, "dur": 2.945,
    "args": {
      "External id": 2483609,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751303.222, "dur": 38.959,
    "args": {
      "External id": 2483610,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751315.765, "dur": 0.994,
    "args": {
      "External id": 2483611,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751348.364, "dur": 31.254,
    "args": {
      "External id": 2483612,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751350.415, "dur": 28.983,
    "args": {
      "External id": 2483613,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751354.312, "dur": 4.172,
    "args": {
      "External id": 2483614,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751359.960, "dur": 18.764,
    "args": {
      "External id": 2483615,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751387.241, "dur": 2.517,
    "args": {
      "External id": 2483616,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751388.587, "dur": 1.079,
    "args": {
      "External id": 2483617,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751396.604, "dur": 4.813,
    "args": {
      "External id": 2483618,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751399.800, "dur": 1.504,
    "args": {
      "External id": 2483619,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751403.257, "dur": 4.564,
    "args": {
      "External id": 2483620,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751404.574, "dur": 3.148,
    "args": {
      "External id": 2483621,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751450.691, "dur": 9.726,
    "args": {
      "External id": 2483622,"Record function id": 0, "Ev Idx": 421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751453.275, "dur": 6.471,
    "args": {
      "External id": 2483623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751455.688, "dur": 3.106,
    "args": {
      "External id": 2483624,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751456.743, "dur": 1.894,
    "args": {
      "External id": 2483625,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751464.321, "dur": 11.099,
    "args": {
      "External id": 2483626,"Record function id": 0, "Sequence number": 24740605, "Fwd thread id": 1, "Ev Idx": 425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751465.469, "dur": 5.778,
    "args": {
      "External id": 2483627,"Sequence number": 24740605, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 426
    }
  },
  {
    "ph": "f", "id": 43, "pid": 1336757, "tid": 1381170, "ts": 1514656751465.469,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751469.020, "dur": 2.032,
    "args": {
      "External id": 2483628,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751469.809, "dur": 1.098,
    "args": {
      "External id": 2483629,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751481.724, "dur": 166.960,
    "args": {
      "External id": 2483630,"Record function id": 0, "Sequence number": 24740604, "Fwd thread id": 1, "Ev Idx": 429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751482.752, "dur": 158.811,
    "args": {
      "External id": 2483631,"Sequence number": 24740604, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 430
    }
  },
  {
    "ph": "f", "id": 44, "pid": 1336757, "tid": 1381170, "ts": 1514656751482.752,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751487.230, "dur": 4.400,
    "args": {
      "External id": 2483632,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751488.821, "dur": 2.187,
    "args": {
      "External id": 2483633,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751490.013, "dur": 0.771,
    "args": {
      "External id": 2483634,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751493.100, "dur": 78.000,
    "args": {
      "External id": 2483635,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751573.682, "dur": 8.448,
    "args": {
      "External id": 2483636,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751576.649, "dur": 4.485,
    "args": {
      "External id": 2483637,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751577.953, "dur": 2.985,
    "args": {
      "External id": 2483638,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751584.092, "dur": 5.411,
    "args": {
      "External id": 2483639,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751585.186, "dur": 3.886,
    "args": {
      "External id": 2483640,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751588.581, "dur": 0.414,
    "args": {
      "External id": 2483641,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751590.048, "dur": 50.711,
    "args": {
      "External id": 2483642,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751655.767, "dur": 13.914,
    "args": {
      "External id": 2483643,"Record function id": 0, "Sequence number": 24740603, "Fwd thread id": 1, "Ev Idx": 442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751656.877, "dur": 9.974,
    "args": {
      "External id": 2483644,"Sequence number": 24740603, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 443
    }
  },
  {
    "ph": "f", "id": 45, "pid": 1336757, "tid": 1381170, "ts": 1514656751656.877,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751661.576, "dur": 5.095,
    "args": {
      "External id": 2483645,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751662.349, "dur": 4.165,
    "args": {
      "External id": 2483646,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751673.295, "dur": 10.359,
    "args": {
      "External id": 2483647,"Record function id": 0, "Sequence number": 24740602, "Fwd thread id": 1, "Ev Idx": 446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751674.349, "dur": 6.099,
    "args": {
      "External id": 2483648,"Sequence number": 24740602, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 447
    }
  },
  {
    "ph": "f", "id": 46, "pid": 1336757, "tid": 1381170, "ts": 1514656751674.349,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751675.389, "dur": 4.838,
    "args": {
      "External id": 2483649,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751676.150, "dur": 3.604,
    "args": {
      "External id": 2483650,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751679.066, "dur": 0.580,
    "args": {
      "External id": 2483651,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751688.138, "dur": 8.727,
    "args": {
      "External id": 2483652,"Record function id": 0, "Ev Idx": 451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751689.464, "dur": 6.819,
    "args": {
      "External id": 2483653,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751691.177, "dur": 4.770,
    "args": {
      "External id": 2483654,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751694.323, "dur": 1.508,
    "args": {
      "External id": 2483655,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751700.616, "dur": 7.692,
    "args": {
      "External id": 2483656,"Record function id": 0, "Sequence number": 24740601, "Fwd thread id": 1, "Ev Idx": 455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751701.596, "dur": 3.763,
    "args": {
      "External id": 2483657,"Sequence number": 24740601, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 456
    }
  },
  {
    "ph": "f", "id": 47, "pid": 1336757, "tid": 1381170, "ts": 1514656751701.596,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751702.836, "dur": 2.360,
    "args": {
      "External id": 2483658,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751703.787, "dur": 1.275,
    "args": {
      "External id": 2483659,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751713.019, "dur": 408.673,
    "args": {
      "External id": 2483660,"Record function id": 0, "Sequence number": 24740600, "Fwd thread id": 1, "Ev Idx": 459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751714.475, "dur": 377.147,
    "args": {
      "External id": 2483661,"Sequence number": 24740600, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 460
    }
  },
  {
    "ph": "f", "id": 48, "pid": 1336757, "tid": 1381170, "ts": 1514656751714.475,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751736.533, "dur": 10.256,
    "args": {
      "External id": 2483662,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751742.838, "dur": 3.579,
    "args": {
      "External id": 2483663,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751748.844, "dur": 3.504,
    "args": {
      "External id": 2483664,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751750.015, "dur": 2.125,
    "args": {
      "External id": 2483665,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751754.062, "dur": 7.874,
    "args": {
      "External id": 2483666,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751757.731, "dur": 3.962,
    "args": {
      "External id": 2483667,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751795.158, "dur": 264.450,
    "args": {
      "External id": 2483668,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751901.333, "dur": 4.135,
    "args": {
      "External id": 2483669,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656751907.429, "dur": 4.768,
    "args": {
      "External id": 2483670,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752077.034, "dur": 4.651,
    "args": {
      "External id": 2483671,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752084.829, "dur": 0.878,
    "args": {
      "External id": 2483672,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752087.826, "dur": 0.618,
    "args": {
      "External id": 2483673,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752133.915, "dur": 271.398,
    "args": {
      "External id": 2483674,"Record function id": 0, "Sequence number": 24740599, "Fwd thread id": 1, "Ev Idx": 473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752135.608, "dur": 259.987,
    "args": {
      "External id": 2483675,"Sequence number": 24740599, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 474
    }
  },
  {
    "ph": "f", "id": 49, "pid": 1336757, "tid": 1381170, "ts": 1514656752135.608,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752160.923, "dur": 57.515,
    "args": {
      "External id": 2483676,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752168.977, "dur": 3.994,
    "args": {
      "External id": 2483677,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752174.210, "dur": 43.490,
    "args": {
      "External id": 2483678,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752230.777, "dur": 4.419,
    "args": {
      "External id": 2483679,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752232.492, "dur": 2.318,
    "args": {
      "External id": 2483680,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752412.744, "dur": 177.726,
    "args": {
      "External id": 2483681,"Record function id": 0, "Sequence number": 24740598, "Fwd thread id": 1, "Ev Idx": 480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752414.561, "dur": 166.978,
    "args": {
      "External id": 2483682,"Sequence number": 24740598, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 481
    }
  },
  {
    "ph": "f", "id": 50, "pid": 1336757, "tid": 1381170, "ts": 1514656752414.561,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752427.554, "dur": 33.799,
    "args": {
      "External id": 2483683,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752431.620, "dur": 2.651,
    "args": {
      "External id": 2483684,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752435.423, "dur": 25.432,
    "args": {
      "External id": 2483685,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752468.207, "dur": 4.024,
    "args": {
      "External id": 2483686,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752469.655, "dur": 2.238,
    "args": {
      "External id": 2483687,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752601.972, "dur": 17.939,
    "args": {
      "External id": 2483688,"Record function id": 0, "Sequence number": 24740597, "Fwd thread id": 1, "Ev Idx": 487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752604.071, "dur": 12.979,
    "args": {
      "External id": 2483689,"Sequence number": 24740597, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 488
    }
  },
  {
    "ph": "f", "id": 51, "pid": 1336757, "tid": 1381170, "ts": 1514656752604.071,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752607.238, "dur": 9.526,
    "args": {
      "External id": 2483690,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752608.657, "dur": 7.901,
    "args": {
      "External id": 2483691,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752623.432, "dur": 9.153,
    "args": {
      "External id": 2483692,"Record function id": 0, "Sequence number": 24740596, "Fwd thread id": 1, "Ev Idx": 491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752624.696, "dur": 5.388,
    "args": {
      "External id": 2483693,"Sequence number": 24740596, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 492
    }
  },
  {
    "ph": "f", "id": 52, "pid": 1336757, "tid": 1381170, "ts": 1514656752624.696,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752628.161, "dur": 1.780,
    "args": {
      "External id": 2483694,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752628.774, "dur": 1.025,
    "args": {
      "External id": 2483695,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752635.812, "dur": 8.777,
    "args": {
      "External id": 2483696,"Record function id": 0, "Sequence number": 24740595, "Fwd thread id": 1, "Ev Idx": 495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752636.899, "dur": 5.751,
    "args": {
      "External id": 2483697,"Sequence number": 24740595, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 496
    }
  },
  {
    "ph": "f", "id": 53, "pid": 1336757, "tid": 1381170, "ts": 1514656752636.899,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752640.825, "dur": 1.698,
    "args": {
      "External id": 2483698,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752641.470, "dur": 0.920,
    "args": {
      "External id": 2483699,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752650.229, "dur": 7.107,
    "args": {
      "External id": 2483700,"Record function id": 0, "Sequence number": 24740594, "Fwd thread id": 1, "Ev Idx": 499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752651.257, "dur": 3.179,
    "args": {
      "External id": 2483701,"Sequence number": 24740594, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 500
    }
  },
  {
    "ph": "f", "id": 54, "pid": 1336757, "tid": 1381170, "ts": 1514656752651.257,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752652.709, "dur": 1.588,
    "args": {
      "External id": 2483702,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752653.126, "dur": 1.061,
    "args": {
      "External id": 2483703,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752660.576, "dur": 193.114,
    "args": {
      "External id": 2483704,"Record function id": 0, "Sequence number": 24740593, "Fwd thread id": 1, "Ev Idx": 503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752661.521, "dur": 183.154,
    "args": {
      "External id": 2483705,"Sequence number": 24740593, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 504
    }
  },
  {
    "ph": "f", "id": 55, "pid": 1336757, "tid": 1381170, "ts": 1514656752661.521,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752666.502, "dur": 9.767,
    "args": {
      "External id": 2483706,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752671.206, "dur": 4.378,
    "args": {
      "External id": 2483707,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752673.523, "dur": 1.736,
    "args": {
      "External id": 2483708,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752678.241, "dur": 73.847,
    "args": {
      "External id": 2483709,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752753.429, "dur": 6.709,
    "args": {
      "External id": 2483710,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752754.244, "dur": 5.079,
    "args": {
      "External id": 2483711,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752757.830, "dur": 1.303,
    "args": {
      "External id": 2483712,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752787.742, "dur": 4.391,
    "args": {
      "External id": 2483713,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752790.501, "dur": 1.161,
    "args": {
      "External id": 2483714,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752791.155, "dur": 0.421,
    "args": {
      "External id": 2483715,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752792.782, "dur": 50.918,
    "args": {
      "External id": 2483716,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752858.852, "dur": 7.698,
    "args": {
      "External id": 2483717,"Record function id": 0, "Sequence number": 24740592, "Fwd thread id": 1, "Ev Idx": 516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752860.055, "dur": 3.957,
    "args": {
      "External id": 2483718,"Sequence number": 24740592, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 517
    }
  },
  {
    "ph": "f", "id": 56, "pid": 1336757, "tid": 1381170, "ts": 1514656752860.055,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752861.792, "dur": 2.064,
    "args": {
      "External id": 2483719,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752862.358, "dur": 1.350,
    "args": {
      "External id": 2483720,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752869.855, "dur": 11.359,
    "args": {
      "External id": 2483721,"Record function id": 0, "Sequence number": 24740591, "Fwd thread id": 1, "Ev Idx": 520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752872.818, "dur": 6.419,
    "args": {
      "External id": 2483722,"Sequence number": 24740591, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 521
    }
  },
  {
    "ph": "f", "id": 57, "pid": 1336757, "tid": 1381170, "ts": 1514656752872.818,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752873.728, "dur": 5.299,
    "args": {
      "External id": 2483723,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752874.619, "dur": 3.897,
    "args": {
      "External id": 2483724,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752877.993, "dur": 0.438,
    "args": {
      "External id": 2483725,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752887.068, "dur": 8.738,
    "args": {
      "External id": 2483726,"Record function id": 0, "Ev Idx": 525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752888.628, "dur": 6.425,
    "args": {
      "External id": 2483727,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752890.812, "dur": 3.850,
    "args": {
      "External id": 2483728,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752891.837, "dur": 2.700,
    "args": {
      "External id": 2483729,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752901.235, "dur": 8.268,
    "args": {
      "External id": 2483730,"Record function id": 0, "Sequence number": 24740590, "Fwd thread id": 1, "Ev Idx": 529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752902.020, "dur": 4.827,
    "args": {
      "External id": 2483731,"Sequence number": 24740590, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 530
    }
  },
  {
    "ph": "f", "id": 58, "pid": 1336757, "tid": 1381170, "ts": 1514656752902.020,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752903.435, "dur": 3.268,
    "args": {
      "External id": 2483732,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752905.757, "dur": 0.849,
    "args": {
      "External id": 2483733,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752912.616, "dur": 150.969,
    "args": {
      "External id": 2483734,"Record function id": 0, "Sequence number": 24740589, "Fwd thread id": 1, "Ev Idx": 533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752913.414, "dur": 141.700,
    "args": {
      "External id": 2483735,"Sequence number": 24740589, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 534
    }
  },
  {
    "ph": "f", "id": 59, "pid": 1336757, "tid": 1381170, "ts": 1514656752913.414,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752916.615, "dur": 4.465,
    "args": {
      "External id": 2483736,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752917.057, "dur": 3.609,
    "args": {
      "External id": 2483737,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752920.080, "dur": 0.486,
    "args": {
      "External id": 2483738,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752924.311, "dur": 30.547,
    "args": {
      "External id": 2483739,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752956.188, "dur": 2.571,
    "args": {
      "External id": 2483740,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752956.786, "dur": 1.386,
    "args": {
      "External id": 2483741,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752957.471, "dur": 0.590,
    "args": {
      "External id": 2483742,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752960.059, "dur": 7.361,
    "args": {
      "External id": 2483743,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752960.761, "dur": 6.130,
    "args": {
      "External id": 2483744,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752964.222, "dur": 2.575,
    "args": {
      "External id": 2483745,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656752968.036, "dur": 85.560,
    "args": {
      "External id": 2483746,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753070.615, "dur": 37.852,
    "args": {
      "External id": 2483747,"Record function id": 0, "Sequence number": 24740588, "Fwd thread id": 1, "Ev Idx": 546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753071.993, "dur": 7.032,
    "args": {
      "External id": 2483748,"Sequence number": 24740588, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 547
    }
  },
  {
    "ph": "f", "id": 60, "pid": 1336757, "tid": 1381170, "ts": 1514656753071.993,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753076.229, "dur": 2.636,
    "args": {
      "External id": 2483749,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753077.088, "dur": 1.673,
    "args": {
      "External id": 2483750,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753082.141, "dur": 23.488,
    "args": {
      "External id": 2483751,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753112.380, "dur": 9.341,
    "args": {
      "External id": 2483752,"Record function id": 0, "Sequence number": 24740587, "Fwd thread id": 1, "Ev Idx": 551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753113.551, "dur": 6.134,
    "args": {
      "External id": 2483753,"Sequence number": 24740587, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 552
    }
  },
  {
    "ph": "f", "id": 61, "pid": 1336757, "tid": 1381170, "ts": 1514656753113.551,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753114.450, "dur": 5.047,
    "args": {
      "External id": 2483754,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753115.191, "dur": 3.765,
    "args": {
      "External id": 2483755,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753118.265, "dur": 0.552,
    "args": {
      "External id": 2483756,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753125.913, "dur": 8.732,
    "args": {
      "External id": 2483757,"Record function id": 0, "Ev Idx": 556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753129.724, "dur": 4.389,
    "args": {
      "External id": 2483758,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753131.245, "dur": 2.400,
    "args": {
      "External id": 2483759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753131.943, "dur": 1.576,
    "args": {
      "External id": 2483760,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753137.755, "dur": 6.863,
    "args": {
      "External id": 2483761,"Record function id": 0, "Sequence number": 24740586, "Fwd thread id": 1, "Ev Idx": 560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753138.879, "dur": 2.940,
    "args": {
      "External id": 2483762,"Sequence number": 24740586, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 561
    }
  },
  {
    "ph": "f", "id": 62, "pid": 1336757, "tid": 1381170, "ts": 1514656753138.879,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753139.860, "dur": 1.817,
    "args": {
      "External id": 2483763,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753140.499, "dur": 1.032,
    "args": {
      "External id": 2483764,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753147.715, "dur": 100.509,
    "args": {
      "External id": 2483765,"Record function id": 0, "Sequence number": 24740585, "Fwd thread id": 1, "Ev Idx": 564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753152.125, "dur": 86.959,
    "args": {
      "External id": 2483766,"Sequence number": 24740585, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 565
    }
  },
  {
    "ph": "f", "id": 63, "pid": 1336757, "tid": 1381170, "ts": 1514656753152.125,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753155.385, "dur": 2.306,
    "args": {
      "External id": 2483767,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753155.809, "dur": 1.472,
    "args": {
      "External id": 2483768,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753156.651, "dur": 0.475,
    "args": {
      "External id": 2483769,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753158.664, "dur": 33.251,
    "args": {
      "External id": 2483770,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753192.983, "dur": 7.748,
    "args": {
      "External id": 2483771,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753193.623, "dur": 6.541,
    "args": {
      "External id": 2483772,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753199.474, "dur": 0.574,
    "args": {
      "External id": 2483773,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753202.133, "dur": 2.225,
    "args": {
      "External id": 2483774,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753202.782, "dur": 1.180,
    "args": {
      "External id": 2483775,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753203.482, "dur": 0.366,
    "args": {
      "External id": 2483776,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753204.808, "dur": 33.484,
    "args": {
      "External id": 2483777,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753252.752, "dur": 28.007,
    "args": {
      "External id": 2483778,"Record function id": 0, "Sequence number": 24740584, "Fwd thread id": 1, "Ev Idx": 577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753253.732, "dur": 5.889,
    "args": {
      "External id": 2483779,"Sequence number": 24740584, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 578
    }
  },
  {
    "ph": "f", "id": 64, "pid": 1336757, "tid": 1381170, "ts": 1514656753253.732,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753257.793, "dur": 1.683,
    "args": {
      "External id": 2483780,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753258.224, "dur": 1.157,
    "args": {
      "External id": 2483781,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753261.716, "dur": 16.851,
    "args": {
      "External id": 2483782,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753284.382, "dur": 11.120,
    "args": {
      "External id": 2483783,"Record function id": 0, "Sequence number": 24740583, "Fwd thread id": 1, "Ev Idx": 582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753285.484, "dur": 7.834,
    "args": {
      "External id": 2483784,"Sequence number": 24740583, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 583
    }
  },
  {
    "ph": "f", "id": 65, "pid": 1336757, "tid": 1381170, "ts": 1514656753285.484,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753286.518, "dur": 6.563,
    "args": {
      "External id": 2483785,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753289.057, "dur": 3.569,
    "args": {
      "External id": 2483786,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753291.967, "dur": 0.517,
    "args": {
      "External id": 2483787,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753299.443, "dur": 7.443,
    "args": {
      "External id": 2483788,"Record function id": 0, "Ev Idx": 587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753300.722, "dur": 5.736,
    "args": {
      "External id": 2483789,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753301.651, "dur": 4.512,
    "args": {
      "External id": 2483790,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753304.882, "dur": 1.151,
    "args": {
      "External id": 2483791,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753311.177, "dur": 408.344,
    "args": {
      "External id": 2483792,"Record function id": 0, "Sequence number": 24740582, "Fwd thread id": 1, "Ev Idx": 591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753312.311, "dur": 369.921,
    "args": {
      "External id": 2483793,"Sequence number": 24740582, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 592
    }
  },
  {
    "ph": "f", "id": 66, "pid": 1336757, "tid": 1381170, "ts": 1514656753312.311,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753348.623, "dur": 1.945,
    "args": {
      "External id": 2483794,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753349.265, "dur": 1.158,
    "args": {
      "External id": 2483795,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753367.555, "dur": 5.672,
    "args": {
      "External id": 2483796,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753383.428, "dur": 1.880,
    "args": {
      "External id": 2483797,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753569.785, "dur": 2.874,
    "args": {
      "External id": 2483798,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753576.990, "dur": 41.927,
    "args": {
      "External id": 2483799,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753591.303, "dur": 1.055,
    "args": {
      "External id": 2483800,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753624.959, "dur": 31.277,
    "args": {
      "External id": 2483801,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753626.641, "dur": 29.353,
    "args": {
      "External id": 2483802,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753630.558, "dur": 5.798,
    "args": {
      "External id": 2483803,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753638.062, "dur": 17.433,
    "args": {
      "External id": 2483804,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753660.914, "dur": 2.454,
    "args": {
      "External id": 2483805,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753662.166, "dur": 1.101,
    "args": {
      "External id": 2483806,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753671.340, "dur": 4.119,
    "args": {
      "External id": 2483807,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753674.370, "dur": 1.002,
    "args": {
      "External id": 2483808,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753694.874, "dur": 18.174,
    "args": {
      "External id": 2483809,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753731.493, "dur": 8.388,
    "args": {
      "External id": 2483810,"Record function id": 0, "Ev Idx": 609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753733.527, "dur": 5.707,
    "args": {
      "External id": 2483811,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753735.505, "dur": 2.614,
    "args": {
      "External id": 2483812,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753736.493, "dur": 1.538,
    "args": {
      "External id": 2483813,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753745.582, "dur": 7.707,
    "args": {
      "External id": 2483814,"Record function id": 0, "Sequence number": 24740581, "Fwd thread id": 1, "Ev Idx": 613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753746.945, "dur": 1.311,
    "args": {
      "External id": 2483815,"Sequence number": 24740581, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 614
    }
  },
  {
    "ph": "f", "id": 67, "pid": 1336757, "tid": 1381170, "ts": 1514656753746.945,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753757.385, "dur": 464.529,
    "args": {
      "External id": 2483816,"Record function id": 0, "Sequence number": 24740580, "Fwd thread id": 1, "Ev Idx": 615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753758.476, "dur": 448.446,
    "args": {
      "External id": 2483817,"Sequence number": 24740580, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 616
    }
  },
  {
    "ph": "f", "id": 68, "pid": 1336757, "tid": 1381170, "ts": 1514656753758.476,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753791.845, "dur": 8.113,
    "args": {
      "External id": 2483818,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753796.596, "dur": 3.127,
    "args": {
      "External id": 2483819,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753803.097, "dur": 6.449,
    "args": {
      "External id": 2483820,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753804.398, "dur": 4.493,
    "args": {
      "External id": 2483821,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753808.036, "dur": 0.673,
    "args": {
      "External id": 2483822,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753815.747, "dur": 82.968,
    "args": {
      "External id": 2483823,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753816.673, "dur": 4.219,
    "args": {
      "External id": 2483824,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753817.249, "dur": 3.157,
    "args": {
      "External id": 2483825,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753817.774, "dur": 2.555,
    "args": {
      "External id": 2483826,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753822.188, "dur": 76.067,
    "args": {
      "External id": 2483827,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753823.506, "dur": 74.063,
    "args": {
      "External id": 2483828,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753902.170, "dur": 5.227,
    "args": {
      "External id": 2483829,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753905.773, "dur": 1.483,
    "args": {
      "External id": 2483830,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753940.093, "dur": 5.395,
    "args": {
      "External id": 2483831,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753946.654, "dur": 2.479,
    "args": {
      "External id": 2483832,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656753949.921, "dur": 1.613,
    "args": {
      "External id": 2483833,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754021.279, "dur": 3.347,
    "args": {
      "External id": 2483834,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754022.284, "dur": 2.088,
    "args": {
      "External id": 2483835,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754050.961, "dur": 135.789,
    "args": {
      "External id": 2483836,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754056.345, "dur": 4.748,
    "args": {
      "External id": 2483837,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754059.271, "dur": 0.947,
    "args": {
      "External id": 2483838,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754062.303, "dur": 11.226,
    "args": {
      "External id": 2483839,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754069.499, "dur": 3.373,
    "args": {
      "External id": 2483840,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754075.074, "dur": 3.462,
    "args": {
      "External id": 2483841,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754077.539, "dur": 0.527,
    "args": {
      "External id": 2483842,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754079.088, "dur": 4.440,
    "args": {
      "External id": 2483843,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754082.412, "dur": 0.516,
    "args": {
      "External id": 2483844,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754087.057, "dur": 1.989,
    "args": {
      "External id": 2483845,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754088.164, "dur": 0.581,
    "args": {
      "External id": 2483846,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754094.443, "dur": 5.916,
    "args": {
      "External id": 2483847,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754097.945, "dur": 2.239,
    "args": {
      "External id": 2483848,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754103.198, "dur": 1.495,
    "args": {
      "External id": 2483849,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754103.869, "dur": 0.573,
    "args": {
      "External id": 2483850,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754105.398, "dur": 5.938,
    "args": {
      "External id": 2483851,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754107.920, "dur": 3.320,
    "args": {
      "External id": 2483852,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754112.630, "dur": 60.092,
    "args": {
      "External id": 2483853,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754174.436, "dur": 3.178,
    "args": {
      "External id": 2483854,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754178.128, "dur": 4.226,
    "args": {
      "External id": 2483855,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754181.349, "dur": 0.426,
    "args": {
      "External id": 2483856,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754184.827, "dur": 0.803,
    "args": {
      "External id": 2483857,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754233.290, "dur": 8.628,
    "args": {
      "External id": 2483858,"Record function id": 0, "Ev Idx": 657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754235.212, "dur": 6.107,
    "args": {
      "External id": 2483859,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754237.287, "dur": 3.221,
    "args": {
      "External id": 2483860,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754238.233, "dur": 2.166,
    "args": {
      "External id": 2483861,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754245.768, "dur": 7.434,
    "args": {
      "External id": 2483862,"Record function id": 0, "Sequence number": 24740579, "Fwd thread id": 1, "Ev Idx": 661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754246.767, "dur": 3.836,
    "args": {
      "External id": 2483863,"Sequence number": 24740579, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 662
    }
  },
  {
    "ph": "f", "id": 69, "pid": 1336757, "tid": 1381170, "ts": 1514656754246.767,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754248.679, "dur": 1.720,
    "args": {
      "External id": 2483864,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754249.496, "dur": 0.777,
    "args": {
      "External id": 2483865,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754256.534, "dur": 129.950,
    "args": {
      "External id": 2483866,"Record function id": 0, "Sequence number": 24740578, "Fwd thread id": 1, "Ev Idx": 665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754257.501, "dur": 120.089,
    "args": {
      "External id": 2483867,"Sequence number": 24740578, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 666
    }
  },
  {
    "ph": "f", "id": 70, "pid": 1336757, "tid": 1381170, "ts": 1514656754257.501,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754263.240, "dur": 6.257,
    "args": {
      "External id": 2483868,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754264.536, "dur": 4.387,
    "args": {
      "External id": 2483869,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754267.939, "dur": 0.784,
    "args": {
      "External id": 2483870,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754270.515, "dur": 43.668,
    "args": {
      "External id": 2483871,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754315.503, "dur": 5.089,
    "args": {
      "External id": 2483872,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754316.023, "dur": 3.854,
    "args": {
      "External id": 2483873,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754319.063, "dur": 0.651,
    "args": {
      "External id": 2483874,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754322.535, "dur": 2.428,
    "args": {
      "External id": 2483875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754323.134, "dur": 1.241,
    "args": {
      "External id": 2483876,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754323.795, "dur": 0.518,
    "args": {
      "External id": 2483877,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754325.390, "dur": 51.341,
    "args": {
      "External id": 2483878,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754391.185, "dur": 7.306,
    "args": {
      "External id": 2483879,"Record function id": 0, "Sequence number": 24740577, "Fwd thread id": 1, "Ev Idx": 678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754392.264, "dur": 5.077,
    "args": {
      "External id": 2483880,"Sequence number": 24740577, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 679
    }
  },
  {
    "ph": "f", "id": 71, "pid": 1336757, "tid": 1381170, "ts": 1514656754392.264,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754395.424, "dur": 1.761,
    "args": {
      "External id": 2483881,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754396.052, "dur": 1.035,
    "args": {
      "External id": 2483882,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754401.817, "dur": 9.819,
    "args": {
      "External id": 2483883,"Record function id": 0, "Sequence number": 24740576, "Fwd thread id": 1, "Ev Idx": 682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754405.068, "dur": 3.948,
    "args": {
      "External id": 2483884,"Sequence number": 24740576, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 683
    }
  },
  {
    "ph": "f", "id": 72, "pid": 1336757, "tid": 1381170, "ts": 1514656754405.068,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754406.228, "dur": 2.553,
    "args": {
      "External id": 2483885,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754406.829, "dur": 1.513,
    "args": {
      "External id": 2483886,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754407.686, "dur": 0.559,
    "args": {
      "External id": 2483887,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754415.543, "dur": 5.454,
    "args": {
      "External id": 2483888,"Record function id": 0, "Ev Idx": 687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754416.995, "dur": 3.522,
    "args": {
      "External id": 2483889,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754418.086, "dur": 2.199,
    "args": {
      "External id": 2483890,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754418.769, "dur": 1.394,
    "args": {
      "External id": 2483891,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754424.088, "dur": 10.482,
    "args": {
      "External id": 2483892,"Record function id": 0, "Sequence number": 24740575, "Fwd thread id": 1, "Ev Idx": 691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754427.133, "dur": 4.825,
    "args": {
      "External id": 2483893,"Sequence number": 24740575, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 692
    }
  },
  {
    "ph": "f", "id": 73, "pid": 1336757, "tid": 1381170, "ts": 1514656754427.133,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754428.329, "dur": 3.487,
    "args": {
      "External id": 2483894,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754431.018, "dur": 0.663,
    "args": {
      "External id": 2483895,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754437.816, "dur": 134.706,
    "args": {
      "External id": 2483896,"Record function id": 0, "Sequence number": 24740574, "Fwd thread id": 1, "Ev Idx": 695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754438.518, "dur": 122.371,
    "args": {
      "External id": 2483897,"Sequence number": 24740574, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 696
    }
  },
  {
    "ph": "f", "id": 74, "pid": 1336757, "tid": 1381170, "ts": 1514656754438.518,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754441.245, "dur": 5.701,
    "args": {
      "External id": 2483898,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754441.767, "dur": 4.746,
    "args": {
      "External id": 2483899,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754445.950, "dur": 0.446,
    "args": {
      "External id": 2483900,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754450.074, "dur": 38.134,
    "args": {
      "External id": 2483901,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754489.331, "dur": 4.872,
    "args": {
      "External id": 2483902,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754490.158, "dur": 3.515,
    "args": {
      "External id": 2483903,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754491.165, "dur": 2.322,
    "args": {
      "External id": 2483904,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754495.494, "dur": 8.804,
    "args": {
      "External id": 2483905,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754496.349, "dur": 7.509,
    "args": {
      "External id": 2483906,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754501.123, "dur": 2.646,
    "args": {
      "External id": 2483907,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754504.780, "dur": 54.869,
    "args": {
      "External id": 2483908,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754579.577, "dur": 33.800,
    "args": {
      "External id": 2483909,"Record function id": 0, "Sequence number": 24740573, "Fwd thread id": 1, "Ev Idx": 708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754580.854, "dur": 4.792,
    "args": {
      "External id": 2483910,"Sequence number": 24740573, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 709
    }
  },
  {
    "ph": "f", "id": 75, "pid": 1336757, "tid": 1381170, "ts": 1514656754580.854,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754583.010, "dur": 2.488,
    "args": {
      "External id": 2483911,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754583.657, "dur": 1.743,
    "args": {
      "External id": 2483912,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754588.320, "dur": 22.330,
    "args": {
      "External id": 2483913,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754617.326, "dur": 11.973,
    "args": {
      "External id": 2483914,"Record function id": 0, "Sequence number": 24740572, "Fwd thread id": 1, "Ev Idx": 713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754618.520, "dur": 8.224,
    "args": {
      "External id": 2483915,"Sequence number": 24740572, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 714
    }
  },
  {
    "ph": "f", "id": 76, "pid": 1336757, "tid": 1381170, "ts": 1514656754618.520,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754619.606, "dur": 6.933,
    "args": {
      "External id": 2483916,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754620.750, "dur": 5.274,
    "args": {
      "External id": 2483917,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754625.315, "dur": 0.623,
    "args": {
      "External id": 2483918,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754633.901, "dur": 5.182,
    "args": {
      "External id": 2483919,"Record function id": 0, "Ev Idx": 718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754635.095, "dur": 3.518,
    "args": {
      "External id": 2483920,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754636.222, "dur": 1.979,
    "args": {
      "External id": 2483921,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754636.864, "dur": 1.258,
    "args": {
      "External id": 2483922,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754644.925, "dur": 460.702,
    "args": {
      "External id": 2483923,"Record function id": 0, "Sequence number": 24740571, "Fwd thread id": 1, "Ev Idx": 722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754646.091, "dur": 417.507,
    "args": {
      "External id": 2483924,"Sequence number": 24740571, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 723
    }
  },
  {
    "ph": "f", "id": 77, "pid": 1336757, "tid": 1381170, "ts": 1514656754646.091,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754668.180, "dur": 35.415,
    "args": {
      "External id": 2483925,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754669.732, "dur": 33.585,
    "args": {
      "External id": 2483926,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754672.474, "dur": 7.896,
    "args": {
      "External id": 2483927,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754676.935, "dur": 2.874,
    "args": {
      "External id": 2483928,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754681.635, "dur": 21.126,
    "args": {
      "External id": 2483929,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754715.361, "dur": 4.619,
    "args": {
      "External id": 2483930,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754718.676, "dur": 1.182,
    "args": {
      "External id": 2483931,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754723.499, "dur": 3.473,
    "args": {
      "External id": 2483932,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754724.048, "dur": 2.831,
    "args": {
      "External id": 2483933,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754738.139, "dur": 1.974,
    "args": {
      "External id": 2483934,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754750.827, "dur": 2.404,
    "args": {
      "External id": 2483935,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754913.108, "dur": 2.431,
    "args": {
      "External id": 2483936,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754919.459, "dur": 32.576,
    "args": {
      "External id": 2483937,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754928.098, "dur": 0.987,
    "args": {
      "External id": 2483938,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754957.363, "dur": 66.834,
    "args": {
      "External id": 2483939,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754958.979, "dur": 64.935,
    "args": {
      "External id": 2483940,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754964.508, "dur": 4.520,
    "args": {
      "External id": 2483941,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656754970.291, "dur": 52.650,
    "args": {
      "External id": 2483942,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755031.007, "dur": 5.126,
    "args": {
      "External id": 2483943,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755034.145, "dur": 1.875,
    "args": {
      "External id": 2483944,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755043.327, "dur": 2.087,
    "args": {
      "External id": 2483945,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755044.019, "dur": 1.292,
    "args": {
      "External id": 2483946,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755049.641, "dur": 3.462,
    "args": {
      "External id": 2483947,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755050.228, "dur": 2.790,
    "args": {
      "External id": 2483948,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755081.745, "dur": 22.289,
    "args": {
      "External id": 2483949,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755118.878, "dur": 11.276,
    "args": {
      "External id": 2483950,"Record function id": 0, "Ev Idx": 749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755121.102, "dur": 8.372,
    "args": {
      "External id": 2483951,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755123.215, "dur": 5.393,
    "args": {
      "External id": 2483952,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755126.654, "dur": 1.875,
    "args": {
      "External id": 2483953,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755133.776, "dur": 8.297,
    "args": {
      "External id": 2483954,"Record function id": 0, "Sequence number": 24740570, "Fwd thread id": 1, "Ev Idx": 753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755134.848, "dur": 4.139,
    "args": {
      "External id": 2483955,"Sequence number": 24740570, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 754
    }
  },
  {
    "ph": "f", "id": 78, "pid": 1336757, "tid": 1381170, "ts": 1514656755134.848,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755136.694, "dur": 2.113,
    "args": {
      "External id": 2483956,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755137.272, "dur": 1.375,
    "args": {
      "External id": 2483957,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755145.535, "dur": 130.293,
    "args": {
      "External id": 2483958,"Record function id": 0, "Sequence number": 24740569, "Fwd thread id": 1, "Ev Idx": 757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755146.630, "dur": 123.153,
    "args": {
      "External id": 2483959,"Sequence number": 24740569, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 758
    }
  },
  {
    "ph": "f", "id": 79, "pid": 1336757, "tid": 1381170, "ts": 1514656755146.630,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755150.000, "dur": 8.621,
    "args": {
      "External id": 2483960,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755153.222, "dur": 4.770,
    "args": {
      "External id": 2483961,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755156.764, "dur": 1.003,
    "args": {
      "External id": 2483962,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755159.717, "dur": 55.766,
    "args": {
      "External id": 2483963,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755216.587, "dur": 5.962,
    "args": {
      "External id": 2483964,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755217.398, "dur": 4.589,
    "args": {
      "External id": 2483965,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755218.476, "dur": 3.345,
    "args": {
      "External id": 2483966,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755225.875, "dur": 2.717,
    "args": {
      "External id": 2483967,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755226.735, "dur": 1.362,
    "args": {
      "External id": 2483968,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755227.692, "dur": 0.341,
    "args": {
      "External id": 2483969,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755229.262, "dur": 39.796,
    "args": {
      "External id": 2483970,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755280.620, "dur": 11.074,
    "args": {
      "External id": 2483971,"Record function id": 0, "Sequence number": 24740568, "Fwd thread id": 1, "Ev Idx": 770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755281.640, "dur": 7.639,
    "args": {
      "External id": 2483972,"Sequence number": 24740568, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 771
    }
  },
  {
    "ph": "f", "id": 80, "pid": 1336757, "tid": 1381170, "ts": 1514656755281.640,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755285.300, "dur": 3.827,
    "args": {
      "External id": 2483973,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755286.157, "dur": 2.842,
    "args": {
      "External id": 2483974,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755297.651, "dur": 10.710,
    "args": {
      "External id": 2483975,"Record function id": 0, "Sequence number": 24740567, "Fwd thread id": 1, "Ev Idx": 774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755300.966, "dur": 4.441,
    "args": {
      "External id": 2483976,"Sequence number": 24740567, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 775
    }
  },
  {
    "ph": "f", "id": 81, "pid": 1336757, "tid": 1381170, "ts": 1514656755300.966,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755302.446, "dur": 2.744,
    "args": {
      "External id": 2483977,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755303.264, "dur": 1.488,
    "args": {
      "External id": 2483978,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755304.299, "dur": 0.293,
    "args": {
      "External id": 2483979,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755312.977, "dur": 4.851,
    "args": {
      "External id": 2483980,"Record function id": 0, "Ev Idx": 779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755314.352, "dur": 2.960,
    "args": {
      "External id": 2483981,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755315.260, "dur": 1.785,
    "args": {
      "External id": 2483982,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755315.835, "dur": 1.122,
    "args": {
      "External id": 2483983,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755320.978, "dur": 10.701,
    "args": {
      "External id": 2483984,"Record function id": 0, "Sequence number": 24740566, "Fwd thread id": 1, "Ev Idx": 783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755322.028, "dur": 7.351,
    "args": {
      "External id": 2483985,"Sequence number": 24740566, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 784
    }
  },
  {
    "ph": "f", "id": 82, "pid": 1336757, "tid": 1381170, "ts": 1514656755322.028,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755325.539, "dur": 3.706,
    "args": {
      "External id": 2483986,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755327.984, "dur": 1.111,
    "args": {
      "External id": 2483987,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755335.848, "dur": 315.954,
    "args": {
      "External id": 2483988,"Record function id": 0, "Sequence number": 24740565, "Fwd thread id": 1, "Ev Idx": 787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755337.202, "dur": 291.071,
    "args": {
      "External id": 2483989,"Sequence number": 24740565, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 788
    }
  },
  {
    "ph": "f", "id": 83, "pid": 1336757, "tid": 1381170, "ts": 1514656755337.202,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755354.766, "dur": 6.450,
    "args": {
      "External id": 2483990,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755357.225, "dur": 3.558,
    "args": {
      "External id": 2483991,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755363.380, "dur": 5.145,
    "args": {
      "External id": 2483992,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755366.603, "dur": 1.694,
    "args": {
      "External id": 2483993,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755370.141, "dur": 4.838,
    "args": {
      "External id": 2483994,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755371.016, "dur": 3.724,
    "args": {
      "External id": 2483995,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755401.461, "dur": 199.777,
    "args": {
      "External id": 2483996,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755487.190, "dur": 3.424,
    "args": {
      "External id": 2483997,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755492.485, "dur": 3.978,
    "args": {
      "External id": 2483998,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755615.202, "dur": 4.235,
    "args": {
      "External id": 2483999,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755622.517, "dur": 0.598,
    "args": {
      "External id": 2484000,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755624.974, "dur": 0.590,
    "args": {
      "External id": 2484001,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755660.704, "dur": 228.920,
    "args": {
      "External id": 2484002,"Record function id": 0, "Sequence number": 24740564, "Fwd thread id": 1, "Ev Idx": 801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755662.282, "dur": 218.703,
    "args": {
      "External id": 2484003,"Sequence number": 24740564, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 802
    }
  },
  {
    "ph": "f", "id": 84, "pid": 1336757, "tid": 1381170, "ts": 1514656755662.282,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755686.597, "dur": 47.288,
    "args": {
      "External id": 2484004,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755689.794, "dur": 3.282,
    "args": {
      "External id": 2484005,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755694.444, "dur": 38.959,
    "args": {
      "External id": 2484006,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755742.888, "dur": 6.066,
    "args": {
      "External id": 2484007,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755746.637, "dur": 1.992,
    "args": {
      "External id": 2484008,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755896.471, "dur": 198.809,
    "args": {
      "External id": 2484009,"Record function id": 0, "Sequence number": 24740563, "Fwd thread id": 1, "Ev Idx": 808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755898.137, "dur": 189.015,
    "args": {
      "External id": 2484010,"Sequence number": 24740563, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 809
    }
  },
  {
    "ph": "f", "id": 85, "pid": 1336757, "tid": 1381170, "ts": 1514656755898.137,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755910.172, "dur": 30.316,
    "args": {
      "External id": 2484011,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755911.853, "dur": 2.447,
    "args": {
      "External id": 2484012,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755915.502, "dur": 24.513,
    "args": {
      "External id": 2484013,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755947.494, "dur": 9.230,
    "args": {
      "External id": 2484014,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656755954.048, "dur": 2.388,
    "args": {
      "External id": 2484015,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756104.493, "dur": 16.250,
    "args": {
      "External id": 2484016,"Record function id": 0, "Sequence number": 24740562, "Fwd thread id": 1, "Ev Idx": 815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756105.999, "dur": 11.171,
    "args": {
      "External id": 2484017,"Sequence number": 24740562, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 816
    }
  },
  {
    "ph": "f", "id": 86, "pid": 1336757, "tid": 1381170, "ts": 1514656756105.999,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756108.704, "dur": 8.107,
    "args": {
      "External id": 2484018,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756109.999, "dur": 6.624,
    "args": {
      "External id": 2484019,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756124.553, "dur": 11.468,
    "args": {
      "External id": 2484020,"Record function id": 0, "Sequence number": 24740561, "Fwd thread id": 1, "Ev Idx": 819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756128.209, "dur": 5.420,
    "args": {
      "External id": 2484021,"Sequence number": 24740561, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 820
    }
  },
  {
    "ph": "f", "id": 87, "pid": 1336757, "tid": 1381170, "ts": 1514656756128.209,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756129.667, "dur": 3.831,
    "args": {
      "External id": 2484022,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756132.378, "dur": 0.994,
    "args": {
      "External id": 2484023,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756141.748, "dur": 5.324,
    "args": {
      "External id": 2484024,"Record function id": 0, "Sequence number": 24740560, "Fwd thread id": 1, "Ev Idx": 823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756142.599, "dur": 2.646,
    "args": {
      "External id": 2484025,"Sequence number": 24740560, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 824
    }
  },
  {
    "ph": "f", "id": 88, "pid": 1336757, "tid": 1381170, "ts": 1514656756142.599,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756143.900, "dur": 1.196,
    "args": {
      "External id": 2484026,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756144.314, "dur": 0.655,
    "args": {
      "External id": 2484027,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756150.432, "dur": 10.996,
    "args": {
      "External id": 2484028,"Record function id": 0, "Sequence number": 24740559, "Fwd thread id": 1, "Ev Idx": 827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756151.264, "dur": 7.760,
    "args": {
      "External id": 2484029,"Sequence number": 24740559, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 828
    }
  },
  {
    "ph": "f", "id": 89, "pid": 1336757, "tid": 1381170, "ts": 1514656756151.264,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756152.822, "dur": 6.044,
    "args": {
      "External id": 2484030,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756157.864, "dur": 0.921,
    "args": {
      "External id": 2484031,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756164.684, "dur": 146.355,
    "args": {
      "External id": 2484032,"Record function id": 0, "Sequence number": 24740558, "Fwd thread id": 1, "Ev Idx": 831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756165.535, "dur": 137.482,
    "args": {
      "External id": 2484033,"Sequence number": 24740558, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 832
    }
  },
  {
    "ph": "f", "id": 90, "pid": 1336757, "tid": 1381170, "ts": 1514656756165.535,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756169.788, "dur": 5.740,
    "args": {
      "External id": 2484034,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756171.552, "dur": 3.350,
    "args": {
      "External id": 2484035,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756173.194, "dur": 1.400,
    "args": {
      "External id": 2484036,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756177.018, "dur": 65.155,
    "args": {
      "External id": 2484037,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756243.446, "dur": 6.518,
    "args": {
      "External id": 2484038,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756246.642, "dur": 2.701,
    "args": {
      "External id": 2484039,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756247.812, "dur": 1.350,
    "args": {
      "External id": 2484040,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756252.019, "dur": 5.426,
    "args": {
      "External id": 2484041,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756253.379, "dur": 3.340,
    "args": {
      "External id": 2484042,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756256.106, "dur": 0.467,
    "args": {
      "External id": 2484043,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756258.148, "dur": 44.018,
    "args": {
      "External id": 2484044,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756316.020, "dur": 9.921,
    "args": {
      "External id": 2484045,"Record function id": 0, "Sequence number": 24740557, "Fwd thread id": 1, "Ev Idx": 844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756317.441, "dur": 6.385,
    "args": {
      "External id": 2484046,"Sequence number": 24740557, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 845
    }
  },
  {
    "ph": "f", "id": 91, "pid": 1336757, "tid": 1381170, "ts": 1514656756317.441,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756321.831, "dur": 1.852,
    "args": {
      "External id": 2484047,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756322.385, "dur": 1.177,
    "args": {
      "External id": 2484048,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756329.314, "dur": 7.723,
    "args": {
      "External id": 2484049,"Record function id": 0, "Sequence number": 24740556, "Fwd thread id": 1, "Ev Idx": 848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756330.169, "dur": 4.443,
    "args": {
      "External id": 2484050,"Sequence number": 24740556, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 849
    }
  },
  {
    "ph": "f", "id": 92, "pid": 1336757, "tid": 1381170, "ts": 1514656756330.169,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756331.674, "dur": 2.712,
    "args": {
      "External id": 2484051,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756332.212, "dur": 1.681,
    "args": {
      "External id": 2484052,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756333.272, "dur": 0.503,
    "args": {
      "External id": 2484053,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756344.921, "dur": 10.962,
    "args": {
      "External id": 2484054,"Record function id": 0, "Ev Idx": 853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756346.577, "dur": 8.577,
    "args": {
      "External id": 2484055,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756349.030, "dur": 5.714,
    "args": {
      "External id": 2484056,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756352.078, "dur": 2.566,
    "args": {
      "External id": 2484057,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756359.126, "dur": 6.798,
    "args": {
      "External id": 2484058,"Record function id": 0, "Sequence number": 24740555, "Fwd thread id": 1, "Ev Idx": 857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756360.307, "dur": 2.913,
    "args": {
      "External id": 2484059,"Sequence number": 24740555, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 858
    }
  },
  {
    "ph": "f", "id": 93, "pid": 1336757, "tid": 1381170, "ts": 1514656756360.307,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756361.601, "dur": 1.461,
    "args": {
      "External id": 2484060,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756362.246, "dur": 0.723,
    "args": {
      "External id": 2484061,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756368.948, "dur": 91.423,
    "args": {
      "External id": 2484062,"Record function id": 0, "Sequence number": 24740554, "Fwd thread id": 1, "Ev Idx": 861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756369.965, "dur": 85.145,
    "args": {
      "External id": 2484063,"Sequence number": 24740554, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 862
    }
  },
  {
    "ph": "f", "id": 94, "pid": 1336757, "tid": 1381170, "ts": 1514656756369.965,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756373.481, "dur": 4.408,
    "args": {
      "External id": 2484064,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756376.281, "dur": 1.197,
    "args": {
      "External id": 2484065,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756376.982, "dur": 0.386,
    "args": {
      "External id": 2484066,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756378.480, "dur": 26.126,
    "args": {
      "External id": 2484067,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756405.658, "dur": 6.494,
    "args": {
      "External id": 2484068,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756406.435, "dur": 5.196,
    "args": {
      "External id": 2484069,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756410.190, "dur": 1.320,
    "args": {
      "External id": 2484070,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756415.252, "dur": 5.448,
    "args": {
      "External id": 2484071,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756415.937, "dur": 4.063,
    "args": {
      "External id": 2484072,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756416.741, "dur": 3.155,
    "args": {
      "External id": 2484073,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756421.224, "dur": 33.279,
    "args": {
      "External id": 2484074,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756464.830, "dur": 34.390,
    "args": {
      "External id": 2484075,"Record function id": 0, "Sequence number": 24740553, "Fwd thread id": 1, "Ev Idx": 874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756465.728, "dur": 5.587,
    "args": {
      "External id": 2484076,"Sequence number": 24740553, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 875
    }
  },
  {
    "ph": "f", "id": 95, "pid": 1336757, "tid": 1381170, "ts": 1514656756465.728,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756467.146, "dur": 4.028,
    "args": {
      "External id": 2484077,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756469.770, "dur": 1.276,
    "args": {
      "External id": 2484078,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756474.208, "dur": 21.473,
    "args": {
      "External id": 2484079,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756503.100, "dur": 7.793,
    "args": {
      "External id": 2484080,"Record function id": 0, "Sequence number": 24740552, "Fwd thread id": 1, "Ev Idx": 879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756504.072, "dur": 4.062,
    "args": {
      "External id": 2484081,"Sequence number": 24740552, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 880
    }
  },
  {
    "ph": "f", "id": 96, "pid": 1336757, "tid": 1381170, "ts": 1514656756504.072,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756505.020, "dur": 2.918,
    "args": {
      "External id": 2484082,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756505.895, "dur": 1.519,
    "args": {
      "External id": 2484083,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756506.838, "dur": 0.422,
    "args": {
      "External id": 2484084,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756514.709, "dur": 7.612,
    "args": {
      "External id": 2484085,"Record function id": 0, "Ev Idx": 884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756515.921, "dur": 5.898,
    "args": {
      "External id": 2484086,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756517.238, "dur": 4.077,
    "args": {
      "External id": 2484087,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756520.189, "dur": 1.029,
    "args": {
      "External id": 2484088,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756525.753, "dur": 26.936,
    "args": {
      "External id": 2484089,"Record function id": 0, "Sequence number": 24740551, "Fwd thread id": 1, "Ev Idx": 888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756526.632, "dur": 22.241,
    "args": {
      "External id": 2484090,"Sequence number": 24740551, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 889
    }
  },
  {
    "ph": "f", "id": 97, "pid": 1336757, "tid": 1381170, "ts": 1514656756526.632,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756542.848, "dur": 5.852,
    "args": {
      "External id": 2484091,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756546.511, "dur": 1.952,
    "args": {
      "External id": 2484092,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756559.722, "dur": 127.273,
    "args": {
      "External id": 2484093,"Record function id": 0, "Sequence number": 24740550, "Fwd thread id": 1, "Ev Idx": 892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756560.950, "dur": 115.520,
    "args": {
      "External id": 2484094,"Sequence number": 24740550, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 893
    }
  },
  {
    "ph": "f", "id": 98, "pid": 1336757, "tid": 1381170, "ts": 1514656756560.950,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756564.503, "dur": 3.994,
    "args": {
      "External id": 2484095,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756565.032, "dur": 3.039,
    "args": {
      "External id": 2484096,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756567.539, "dur": 0.417,
    "args": {
      "External id": 2484097,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756569.089, "dur": 36.920,
    "args": {
      "External id": 2484098,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756606.970, "dur": 3.792,
    "args": {
      "External id": 2484099,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756607.703, "dur": 2.504,
    "args": {
      "External id": 2484100,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756608.507, "dur": 1.567,
    "args": {
      "External id": 2484101,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756612.223, "dur": 9.711,
    "args": {
      "External id": 2484102,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756617.776, "dur": 3.794,
    "args": {
      "External id": 2484103,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756621.091, "dur": 0.349,
    "args": {
      "External id": 2484104,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756622.409, "dur": 53.256,
    "args": {
      "External id": 2484105,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756691.672, "dur": 28.177,
    "args": {
      "External id": 2484106,"Record function id": 0, "Sequence number": 24740549, "Fwd thread id": 1, "Ev Idx": 905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756692.828, "dur": 3.514,
    "args": {
      "External id": 2484107,"Sequence number": 24740549, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 906
    }
  },
  {
    "ph": "f", "id": 99, "pid": 1336757, "tid": 1381170, "ts": 1514656756692.828,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756694.033, "dur": 2.171,
    "args": {
      "External id": 2484108,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756694.912, "dur": 1.198,
    "args": {
      "External id": 2484109,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756699.224, "dur": 18.367,
    "args": {
      "External id": 2484110,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756723.739, "dur": 12.803,
    "args": {
      "External id": 2484111,"Record function id": 0, "Sequence number": 24740548, "Fwd thread id": 1, "Ev Idx": 910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756727.588, "dur": 6.649,
    "args": {
      "External id": 2484112,"Sequence number": 24740548, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 911
    }
  },
  {
    "ph": "f", "id": 100, "pid": 1336757, "tid": 1381170, "ts": 1514656756727.588,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756728.819, "dur": 5.216,
    "args": {
      "External id": 2484113,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756729.674, "dur": 3.888,
    "args": {
      "External id": 2484114,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756732.966, "dur": 0.481,
    "args": {
      "External id": 2484115,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756740.848, "dur": 5.308,
    "args": {
      "External id": 2484116,"Record function id": 0, "Ev Idx": 915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756742.038, "dur": 3.649,
    "args": {
      "External id": 2484117,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756743.288, "dur": 2.084,
    "args": {
      "External id": 2484118,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756743.984, "dur": 1.252,
    "args": {
      "External id": 2484119,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756750.377, "dur": 393.231,
    "args": {
      "External id": 2484120,"Record function id": 0, "Sequence number": 24740547, "Fwd thread id": 1, "Ev Idx": 919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756751.574, "dur": 356.544,
    "args": {
      "External id": 2484121,"Sequence number": 24740547, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 920
    }
  },
  {
    "ph": "f", "id": 101, "pid": 1336757, "tid": 1381170, "ts": 1514656756751.574,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756785.622, "dur": 3.990,
    "args": {
      "External id": 2484122,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756788.204, "dur": 1.252,
    "args": {
      "External id": 2484123,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756803.466, "dur": 5.868,
    "args": {
      "External id": 2484124,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756818.059, "dur": 2.839,
    "args": {
      "External id": 2484125,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756958.840, "dur": 1.831,
    "args": {
      "External id": 2484126,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756964.913, "dur": 71.537,
    "args": {
      "External id": 2484127,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656756974.684, "dur": 0.940,
    "args": {
      "External id": 2484128,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757047.141, "dur": 37.457,
    "args": {
      "External id": 2484129,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757048.908, "dur": 35.490,
    "args": {
      "External id": 2484130,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757057.197, "dur": 6.825,
    "args": {
      "External id": 2484131,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757065.519, "dur": 18.380,
    "args": {
      "External id": 2484132,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757089.376, "dur": 3.094,
    "args": {
      "External id": 2484133,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757090.396, "dur": 1.966,
    "args": {
      "External id": 2484134,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757098.839, "dur": 2.539,
    "args": {
      "External id": 2484135,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757099.918, "dur": 1.370,
    "args": {
      "External id": 2484136,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757123.057, "dur": 16.008,
    "args": {
      "External id": 2484137,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757155.938, "dur": 10.675,
    "args": {
      "External id": 2484138,"Record function id": 0, "Ev Idx": 937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757159.575, "dur": 6.344,
    "args": {
      "External id": 2484139,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757161.673, "dur": 3.063,
    "args": {
      "External id": 2484140,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757162.714, "dur": 1.905,
    "args": {
      "External id": 2484141,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757170.543, "dur": 6.884,
    "args": {
      "External id": 2484142,"Record function id": 0, "Sequence number": 24740546, "Fwd thread id": 1, "Ev Idx": 941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757172.183, "dur": 1.333,
    "args": {
      "External id": 2484143,"Sequence number": 24740546, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 942
    }
  },
  {
    "ph": "f", "id": 102, "pid": 1336757, "tid": 1381170, "ts": 1514656757172.183,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757181.336, "dur": 453.655,
    "args": {
      "External id": 2484144,"Record function id": 0, "Sequence number": 24740545, "Fwd thread id": 1, "Ev Idx": 943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757182.515, "dur": 438.541,
    "args": {
      "External id": 2484145,"Sequence number": 24740545, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 944
    }
  },
  {
    "ph": "f", "id": 103, "pid": 1336757, "tid": 1381170, "ts": 1514656757182.515,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757214.794, "dur": 7.338,
    "args": {
      "External id": 2484146,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757218.954, "dur": 2.924,
    "args": {
      "External id": 2484147,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757225.336, "dur": 7.730,
    "args": {
      "External id": 2484148,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757227.109, "dur": 5.204,
    "args": {
      "External id": 2484149,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757231.243, "dur": 0.885,
    "args": {
      "External id": 2484150,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757236.618, "dur": 99.815,
    "args": {
      "External id": 2484151,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757237.673, "dur": 6.350,
    "args": {
      "External id": 2484152,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757240.189, "dur": 3.398,
    "args": {
      "External id": 2484153,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757240.794, "dur": 2.651,
    "args": {
      "External id": 2484154,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757245.231, "dur": 90.706,
    "args": {
      "External id": 2484155,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757246.740, "dur": 88.526,
    "args": {
      "External id": 2484156,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757340.424, "dur": 2.823,
    "args": {
      "External id": 2484157,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757341.555, "dur": 1.564,
    "args": {
      "External id": 2484158,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757376.454, "dur": 4.884,
    "args": {
      "External id": 2484159,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757384.858, "dur": 2.186,
    "args": {
      "External id": 2484160,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757387.664, "dur": 2.308,
    "args": {
      "External id": 2484161,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757423.848, "dur": 2.078,
    "args": {
      "External id": 2484162,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757424.330, "dur": 1.439,
    "args": {
      "External id": 2484163,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757448.194, "dur": 152.809,
    "args": {
      "External id": 2484164,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757452.861, "dur": 7.214,
    "args": {
      "External id": 2484165,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757458.347, "dur": 0.863,
    "args": {
      "External id": 2484166,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757461.468, "dur": 10.503,
    "args": {
      "External id": 2484167,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757468.645, "dur": 2.644,
    "args": {
      "External id": 2484168,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757473.352, "dur": 1.363,
    "args": {
      "External id": 2484169,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757473.953, "dur": 0.408,
    "args": {
      "External id": 2484170,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757477.008, "dur": 2.044,
    "args": {
      "External id": 2484171,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757478.057, "dur": 0.531,
    "args": {
      "External id": 2484172,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757484.323, "dur": 1.747,
    "args": {
      "External id": 2484173,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757485.329, "dur": 0.466,
    "args": {
      "External id": 2484174,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757487.094, "dur": 7.504,
    "args": {
      "External id": 2484175,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757492.452, "dur": 1.956,
    "args": {
      "External id": 2484176,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757495.452, "dur": 3.659,
    "args": {
      "External id": 2484177,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757498.270, "dur": 0.575,
    "args": {
      "External id": 2484178,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757501.798, "dur": 3.346,
    "args": {
      "External id": 2484179,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757502.354, "dur": 2.697,
    "args": {
      "External id": 2484180,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757506.421, "dur": 76.686,
    "args": {
      "External id": 2484181,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757587.407, "dur": 3.951,
    "args": {
      "External id": 2484182,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757592.133, "dur": 4.694,
    "args": {
      "External id": 2484183,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757595.447, "dur": 0.687,
    "args": {
      "External id": 2484184,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757599.153, "dur": 0.845,
    "args": {
      "External id": 2484185,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757647.274, "dur": 9.670,
    "args": {
      "External id": 2484186,"Record function id": 0, "Ev Idx": 985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757649.729, "dur": 6.611,
    "args": {
      "External id": 2484187,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757651.762, "dur": 3.614,
    "args": {
      "External id": 2484188,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757652.945, "dur": 2.306,
    "args": {
      "External id": 2484189,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757660.997, "dur": 9.085,
    "args": {
      "External id": 2484190,"Record function id": 0, "Sequence number": 24740544, "Fwd thread id": 1, "Ev Idx": 989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757661.945, "dur": 5.848,
    "args": {
      "External id": 2484191,"Sequence number": 24740544, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 990
    }
  },
  {
    "ph": "f", "id": 104, "pid": 1336757, "tid": 1381170, "ts": 1514656757661.945,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757666.042, "dur": 1.567,
    "args": {
      "External id": 2484192,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757666.610, "dur": 0.907,
    "args": {
      "External id": 2484193,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757673.812, "dur": 130.391,
    "args": {
      "External id": 2484194,"Record function id": 0, "Sequence number": 24740543, "Fwd thread id": 1, "Ev Idx": 993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757674.820, "dur": 121.177,
    "args": {
      "External id": 2484195,"Sequence number": 24740543, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 994
    }
  },
  {
    "ph": "f", "id": 105, "pid": 1336757, "tid": 1381170, "ts": 1514656757674.820,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757678.451, "dur": 6.549,
    "args": {
      "External id": 2484196,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757680.025, "dur": 4.473,
    "args": {
      "External id": 2484197,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757683.489, "dur": 0.853,
    "args": {
      "External id": 2484198,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757686.218, "dur": 44.875,
    "args": {
      "External id": 2484199,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757732.299, "dur": 5.709,
    "args": {
      "External id": 2484200,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757732.891, "dur": 4.257,
    "args": {
      "External id": 2484201,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757736.101, "dur": 0.928,
    "args": {
      "External id": 2484202,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757739.526, "dur": 5.023,
    "args": {
      "External id": 2484203,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757740.685, "dur": 3.234,
    "args": {
      "External id": 2484204,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757743.190, "dur": 0.660,
    "args": {
      "External id": 2484205,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757745.086, "dur": 50.092,
    "args": {
      "External id": 2484206,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757808.779, "dur": 7.901,
    "args": {
      "External id": 2484207,"Record function id": 0, "Sequence number": 24740542, "Fwd thread id": 1, "Ev Idx": 1006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757809.792, "dur": 5.715,
    "args": {
      "External id": 2484208,"Sequence number": 24740542, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1007
    }
  },
  {
    "ph": "f", "id": 106, "pid": 1336757, "tid": 1381170, "ts": 1514656757809.792,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757811.369, "dur": 3.999,
    "args": {
      "External id": 2484209,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757814.154, "dur": 1.095,
    "args": {
      "External id": 2484210,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757819.977, "dur": 8.925,
    "args": {
      "External id": 2484211,"Record function id": 0, "Sequence number": 24740541, "Fwd thread id": 1, "Ev Idx": 1010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757820.818, "dur": 5.770,
    "args": {
      "External id": 2484212,"Sequence number": 24740541, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1011
    }
  },
  {
    "ph": "f", "id": 107, "pid": 1336757, "tid": 1381170, "ts": 1514656757820.818,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757821.897, "dur": 4.471,
    "args": {
      "External id": 2484213,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757822.702, "dur": 3.194,
    "args": {
      "External id": 2484214,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757825.119, "dur": 0.662,
    "args": {
      "External id": 2484215,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757832.718, "dur": 4.932,
    "args": {
      "External id": 2484216,"Record function id": 0, "Ev Idx": 1015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757833.788, "dur": 3.401,
    "args": {
      "External id": 2484217,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757834.923, "dur": 1.970,
    "args": {
      "External id": 2484218,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757835.579, "dur": 1.236,
    "args": {
      "External id": 2484219,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757843.021, "dur": 8.384,
    "args": {
      "External id": 2484220,"Record function id": 0, "Sequence number": 24740540, "Fwd thread id": 1, "Ev Idx": 1019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757843.896, "dur": 5.096,
    "args": {
      "External id": 2484221,"Sequence number": 24740540, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 1020
    }
  },
  {
    "ph": "f", "id": 108, "pid": 1336757, "tid": 1381170, "ts": 1514656757843.896,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757844.974, "dur": 3.865,
    "args": {
      "External id": 2484222,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757847.956, "dur": 0.792,
    "args": {
      "External id": 2484223,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757854.560, "dur": 187.412,
    "args": {
      "External id": 2484224,"Record function id": 0, "Sequence number": 24740539, "Fwd thread id": 1, "Ev Idx": 1023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757876.990, "dur": 105.875,
    "args": {
      "External id": 2484225,"Sequence number": 24740539, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1024
    }
  },
  {
    "ph": "f", "id": 109, "pid": 1336757, "tid": 1381170, "ts": 1514656757876.990,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757881.376, "dur": 2.170,
    "args": {
      "External id": 2484226,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757881.872, "dur": 1.280,
    "args": {
      "External id": 2484227,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 1026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757882.538, "dur": 0.508,
    "args": {
      "External id": 2484228,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757884.203, "dur": 33.749,
    "args": {
      "External id": 2484229,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 1028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757919.196, "dur": 7.558,
    "args": {
      "External id": 2484230,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757919.676, "dur": 6.527,
    "args": {
      "External id": 2484231,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757923.111, "dur": 2.945,
    "args": {
      "External id": 2484232,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757927.998, "dur": 6.650,
    "args": {
      "External id": 2484233,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757929.121, "dur": 5.073,
    "args": {
      "External id": 2484234,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757931.677, "dur": 2.452,
    "args": {
      "External id": 2484235,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656757935.078, "dur": 47.089,
    "args": {
      "External id": 2484236,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758049.491, "dur": 40.054,
    "args": {
      "External id": 2484237,"Record function id": 0, "Sequence number": 24740538, "Fwd thread id": 1, "Ev Idx": 1036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758050.933, "dur": 7.009,
    "args": {
      "External id": 2484238,"Sequence number": 24740538, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1037
    }
  },
  {
    "ph": "f", "id": 110, "pid": 1336757, "tid": 1381170, "ts": 1514656758050.933,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758055.263, "dur": 2.490,
    "args": {
      "External id": 2484239,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758055.742, "dur": 1.775,
    "args": {
      "External id": 2484240,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758060.934, "dur": 25.704,
    "args": {
      "External id": 2484241,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758093.349, "dur": 9.938,
    "args": {
      "External id": 2484242,"Record function id": 0, "Sequence number": 24740537, "Fwd thread id": 1, "Ev Idx": 1041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758094.323, "dur": 6.875,
    "args": {
      "External id": 2484243,"Sequence number": 24740537, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1042
    }
  },
  {
    "ph": "f", "id": 111, "pid": 1336757, "tid": 1381170, "ts": 1514656758094.323,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758095.259, "dur": 5.708,
    "args": {
      "External id": 2484244,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758096.217, "dur": 4.178,
    "args": {
      "External id": 2484245,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758099.717, "dur": 0.521,
    "args": {
      "External id": 2484246,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758107.537, "dur": 8.193,
    "args": {
      "External id": 2484247,"Record function id": 0, "Ev Idx": 1046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758108.861, "dur": 6.348,
    "args": {
      "External id": 2484248,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758109.868, "dur": 4.922,
    "args": {
      "External id": 2484249,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758113.195, "dur": 1.494,
    "args": {
      "External id": 2484250,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758119.514, "dur": 435.643,
    "args": {
      "External id": 2484251,"Record function id": 0, "Sequence number": 24740536, "Fwd thread id": 1, "Ev Idx": 1050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758120.754, "dur": 380.143,
    "args": {
      "External id": 2484252,"Sequence number": 24740536, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 1051
    }
  },
  {
    "ph": "f", "id": 112, "pid": 1336757, "tid": 1381170, "ts": 1514656758120.754,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758143.910, "dur": 32.993,
    "args": {
      "External id": 2484253,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758145.218, "dur": 31.505,
    "args": {
      "External id": 2484254,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758147.661, "dur": 5.884,
    "args": {
      "External id": 2484255,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 1054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758150.193, "dur": 2.875,
    "args": {
      "External id": 2484256,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758154.639, "dur": 21.636,
    "args": {
      "External id": 2484257,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758190.872, "dur": 2.244,
    "args": {
      "External id": 2484258,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758191.626, "dur": 1.386,
    "args": {
      "External id": 2484259,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758197.079, "dur": 5.917,
    "args": {
      "External id": 2484260,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758200.226, "dur": 2.606,
    "args": {
      "External id": 2484261,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758214.908, "dur": 2.335,
    "args": {
      "External id": 2484262,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758227.745, "dur": 2.061,
    "args": {
      "External id": 2484263,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758388.341, "dur": 2.263,
    "args": {
      "External id": 2484264,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758397.348, "dur": 32.511,
    "args": {
      "External id": 2484265,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758406.705, "dur": 0.757,
    "args": {
      "External id": 2484266,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758435.465, "dur": 31.979,
    "args": {
      "External id": 2484267,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758439.643, "dur": 27.547,
    "args": {
      "External id": 2484268,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758445.484, "dur": 4.730,
    "args": {
      "External id": 2484269,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758451.428, "dur": 15.223,
    "args": {
      "External id": 2484270,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758471.904, "dur": 2.376,
    "args": {
      "External id": 2484271,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758472.919, "dur": 1.220,
    "args": {
      "External id": 2484272,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758480.008, "dur": 3.798,
    "args": {
      "External id": 2484273,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758482.637, "dur": 1.066,
    "args": {
      "External id": 2484274,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758485.475, "dur": 6.213,
    "args": {
      "External id": 2484275,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758488.223, "dur": 3.379,
    "args": {
      "External id": 2484276,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758516.029, "dur": 36.497,
    "args": {
      "External id": 2484277,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758568.218, "dur": 9.322,
    "args": {
      "External id": 2484278,"Record function id": 0, "Ev Idx": 1077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758570.482, "dur": 6.418,
    "args": {
      "External id": 2484279,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758572.592, "dur": 3.080,
    "args": {
      "External id": 2484280,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758573.450, "dur": 2.130,
    "args": {
      "External id": 2484281,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758580.976, "dur": 10.138,
    "args": {
      "External id": 2484282,"Record function id": 0, "Sequence number": 24740535, "Fwd thread id": 1, "Ev Idx": 1081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758581.866, "dur": 6.681,
    "args": {
      "External id": 2484283,"Sequence number": 24740535, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1082
    }
  },
  {
    "ph": "f", "id": 113, "pid": 1336757, "tid": 1381170, "ts": 1514656758581.866,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758585.883, "dur": 2.452,
    "args": {
      "External id": 2484284,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758586.700, "dur": 1.470,
    "args": {
      "External id": 2484285,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758594.570, "dur": 130.463,
    "args": {
      "External id": 2484286,"Record function id": 0, "Sequence number": 24740534, "Fwd thread id": 1, "Ev Idx": 1085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758595.524, "dur": 122.478,
    "args": {
      "External id": 2484287,"Sequence number": 24740534, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1086
    }
  },
  {
    "ph": "f", "id": 114, "pid": 1336757, "tid": 1381170, "ts": 1514656758595.524,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758599.525, "dur": 6.135,
    "args": {
      "External id": 2484288,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758600.724, "dur": 4.339,
    "args": {
      "External id": 2484289,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758604.010, "dur": 0.854,
    "args": {
      "External id": 2484290,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758606.964, "dur": 55.277,
    "args": {
      "External id": 2484291,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758663.428, "dur": 7.544,
    "args": {
      "External id": 2484292,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758664.024, "dur": 6.343,
    "args": {
      "External id": 2484293,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758667.257, "dur": 2.938,
    "args": {
      "External id": 2484294,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758672.694, "dur": 5.341,
    "args": {
      "External id": 2484295,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758674.014, "dur": 3.578,
    "args": {
      "External id": 2484296,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758677.020, "dur": 0.498,
    "args": {
      "External id": 2484297,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758678.598, "dur": 38.514,
    "args": {
      "External id": 2484298,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758729.534, "dur": 12.501,
    "args": {
      "External id": 2484299,"Record function id": 0, "Sequence number": 24740533, "Fwd thread id": 1, "Ev Idx": 1098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758730.742, "dur": 8.087,
    "args": {
      "External id": 2484300,"Sequence number": 24740533, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1099
    }
  },
  {
    "ph": "f", "id": 115, "pid": 1336757, "tid": 1381170, "ts": 1514656758730.742,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758732.241, "dur": 6.414,
    "args": {
      "External id": 2484301,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758735.704, "dur": 2.805,
    "args": {
      "External id": 2484302,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758748.170, "dur": 9.114,
    "args": {
      "External id": 2484303,"Record function id": 0, "Sequence number": 24740532, "Fwd thread id": 1, "Ev Idx": 1102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758748.890, "dur": 6.393,
    "args": {
      "External id": 2484304,"Sequence number": 24740532, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1103
    }
  },
  {
    "ph": "f", "id": 116, "pid": 1336757, "tid": 1381170, "ts": 1514656758748.890,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758750.073, "dur": 4.966,
    "args": {
      "External id": 2484305,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758750.695, "dur": 3.883,
    "args": {
      "External id": 2484306,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758753.942, "dur": 0.492,
    "args": {
      "External id": 2484307,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758761.310, "dur": 4.987,
    "args": {
      "External id": 2484308,"Record function id": 0, "Ev Idx": 1107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758762.590, "dur": 3.183,
    "args": {
      "External id": 2484309,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758763.653, "dur": 1.673,
    "args": {
      "External id": 2484310,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758764.219, "dur": 0.996,
    "args": {
      "External id": 2484311,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758771.743, "dur": 5.814,
    "args": {
      "External id": 2484312,"Record function id": 0, "Sequence number": 24740531, "Fwd thread id": 1, "Ev Idx": 1111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758772.746, "dur": 3.019,
    "args": {
      "External id": 2484313,"Sequence number": 24740531, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1112
    }
  },
  {
    "ph": "f", "id": 117, "pid": 1336757, "tid": 1381170, "ts": 1514656758772.746,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758773.956, "dur": 1.665,
    "args": {
      "External id": 2484314,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758774.341, "dur": 1.136,
    "args": {
      "External id": 2484315,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758783.746, "dur": 348.313,
    "args": {
      "External id": 2484316,"Record function id": 0, "Sequence number": 24740530, "Fwd thread id": 1, "Ev Idx": 1115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758784.866, "dur": 320.991,
    "args": {
      "External id": 2484317,"Sequence number": 24740530, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1116
    }
  },
  {
    "ph": "f", "id": 118, "pid": 1336757, "tid": 1381170, "ts": 1514656758784.866,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758802.288, "dur": 6.111,
    "args": {
      "External id": 2484318,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758804.295, "dur": 3.673,
    "args": {
      "External id": 2484319,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758810.166, "dur": 6.544,
    "args": {
      "External id": 2484320,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758812.577, "dur": 3.899,
    "args": {
      "External id": 2484321,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758824.968, "dur": 4.463,
    "args": {
      "External id": 2484322,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758827.520, "dur": 1.700,
    "args": {
      "External id": 2484323,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758858.532, "dur": 220.033,
    "args": {
      "External id": 2484324,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758939.856, "dur": 5.441,
    "args": {
      "External id": 2484325,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656758946.988, "dur": 4.333,
    "args": {
      "External id": 2484326,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759093.005, "dur": 4.143,
    "args": {
      "External id": 2484327,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759099.824, "dur": 0.898,
    "args": {
      "External id": 2484328,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759102.599, "dur": 0.592,
    "args": {
      "External id": 2484329,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759142.151, "dur": 235.727,
    "args": {
      "External id": 2484330,"Record function id": 0, "Sequence number": 24740529, "Fwd thread id": 1, "Ev Idx": 1129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759143.698, "dur": 227.382,
    "args": {
      "External id": 2484331,"Sequence number": 24740529, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1130
    }
  },
  {
    "ph": "f", "id": 119, "pid": 1336757, "tid": 1381170, "ts": 1514656759143.698,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759164.428, "dur": 47.139,
    "args": {
      "External id": 2484332,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759167.536, "dur": 3.874,
    "args": {
      "External id": 2484333,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759172.503, "dur": 38.441,
    "args": {
      "External id": 2484334,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759221.800, "dur": 6.966,
    "args": {
      "External id": 2484335,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759225.603, "dur": 2.840,
    "args": {
      "External id": 2484336,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759384.953, "dur": 180.251,
    "args": {
      "External id": 2484337,"Record function id": 0, "Sequence number": 24740528, "Fwd thread id": 1, "Ev Idx": 1136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759386.433, "dur": 171.119,
    "args": {
      "External id": 2484338,"Sequence number": 24740528, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1137
    }
  },
  {
    "ph": "f", "id": 120, "pid": 1336757, "tid": 1381170, "ts": 1514656759386.433,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759397.978, "dur": 42.901,
    "args": {
      "External id": 2484339,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759400.250, "dur": 2.608,
    "args": {
      "External id": 2484340,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759409.135, "dur": 31.286,
    "args": {
      "External id": 2484341,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759448.002, "dur": 5.904,
    "args": {
      "External id": 2484342,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759451.576, "dur": 2.001,
    "args": {
      "External id": 2484343,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759573.825, "dur": 19.740,
    "args": {
      "External id": 2484344,"Record function id": 0, "Sequence number": 24740527, "Fwd thread id": 1, "Ev Idx": 1143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759575.355, "dur": 15.279,
    "args": {
      "External id": 2484345,"Sequence number": 24740527, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1144
    }
  },
  {
    "ph": "f", "id": 121, "pid": 1336757, "tid": 1381170, "ts": 1514656759575.355,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759578.109, "dur": 12.138,
    "args": {
      "External id": 2484346,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759581.418, "dur": 8.627,
    "args": {
      "External id": 2484347,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759597.088, "dur": 8.715,
    "args": {
      "External id": 2484348,"Record function id": 0, "Sequence number": 24740526, "Fwd thread id": 1, "Ev Idx": 1147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759598.024, "dur": 5.542,
    "args": {
      "External id": 2484349,"Sequence number": 24740526, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1148
    }
  },
  {
    "ph": "f", "id": 122, "pid": 1336757, "tid": 1381170, "ts": 1514656759598.024,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759599.021, "dur": 4.392,
    "args": {
      "External id": 2484350,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759602.416, "dur": 0.873,
    "args": {
      "External id": 2484351,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759608.765, "dur": 7.469,
    "args": {
      "External id": 2484352,"Record function id": 0, "Sequence number": 24740525, "Fwd thread id": 1, "Ev Idx": 1151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759610.341, "dur": 3.038,
    "args": {
      "External id": 2484353,"Sequence number": 24740525, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1152
    }
  },
  {
    "ph": "f", "id": 123, "pid": 1336757, "tid": 1381170, "ts": 1514656759610.341,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759611.504, "dur": 1.724,
    "args": {
      "External id": 2484354,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759612.219, "dur": 0.891,
    "args": {
      "External id": 2484355,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759619.574, "dur": 10.173,
    "args": {
      "External id": 2484356,"Record function id": 0, "Sequence number": 24740524, "Fwd thread id": 1, "Ev Idx": 1155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759620.518, "dur": 6.997,
    "args": {
      "External id": 2484357,"Sequence number": 24740524, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1156
    }
  },
  {
    "ph": "f", "id": 124, "pid": 1336757, "tid": 1381170, "ts": 1514656759620.518,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759621.765, "dur": 5.611,
    "args": {
      "External id": 2484358,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759625.920, "dur": 1.359,
    "args": {
      "External id": 2484359,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759632.898, "dur": 153.600,
    "args": {
      "External id": 2484360,"Record function id": 0, "Sequence number": 24740523, "Fwd thread id": 1, "Ev Idx": 1159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759634.025, "dur": 144.744,
    "args": {
      "External id": 2484361,"Sequence number": 24740523, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1160
    }
  },
  {
    "ph": "f", "id": 125, "pid": 1336757, "tid": 1381170, "ts": 1514656759634.025,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759638.825, "dur": 6.206,
    "args": {
      "External id": 2484362,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759640.576, "dur": 3.810,
    "args": {
      "External id": 2484363,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759642.669, "dur": 1.442,
    "args": {
      "External id": 2484364,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759649.183, "dur": 68.850,
    "args": {
      "External id": 2484365,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759719.189, "dur": 4.509,
    "args": {
      "External id": 2484366,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759720.126, "dur": 2.714,
    "args": {
      "External id": 2484367,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759721.202, "dur": 1.484,
    "args": {
      "External id": 2484368,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759725.782, "dur": 5.559,
    "args": {
      "External id": 2484369,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759727.388, "dur": 3.422,
    "args": {
      "External id": 2484370,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759730.304, "dur": 0.392,
    "args": {
      "External id": 2484371,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759733.762, "dur": 44.171,
    "args": {
      "External id": 2484372,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759793.717, "dur": 7.331,
    "args": {
      "External id": 2484373,"Record function id": 0, "Sequence number": 24740522, "Fwd thread id": 1, "Ev Idx": 1172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759794.784, "dur": 4.106,
    "args": {
      "External id": 2484374,"Sequence number": 24740522, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1173
    }
  },
  {
    "ph": "f", "id": 126, "pid": 1336757, "tid": 1381170, "ts": 1514656759794.784,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759796.577, "dur": 2.149,
    "args": {
      "External id": 2484375,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759797.279, "dur": 1.303,
    "args": {
      "External id": 2484376,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759804.615, "dur": 9.529,
    "args": {
      "External id": 2484377,"Record function id": 0, "Sequence number": 24740521, "Fwd thread id": 1, "Ev Idx": 1176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759805.422, "dur": 5.580,
    "args": {
      "External id": 2484378,"Sequence number": 24740521, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1177
    }
  },
  {
    "ph": "f", "id": 127, "pid": 1336757, "tid": 1381170, "ts": 1514656759805.422,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759806.505, "dur": 4.281,
    "args": {
      "External id": 2484379,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759807.018, "dur": 3.295,
    "args": {
      "External id": 2484380,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759809.742, "dur": 0.444,
    "args": {
      "External id": 2484381,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759822.197, "dur": 9.071,
    "args": {
      "External id": 2484382,"Record function id": 0, "Ev Idx": 1181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759823.914, "dur": 6.708,
    "args": {
      "External id": 2484383,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759826.429, "dur": 3.764,
    "args": {
      "External id": 2484384,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759827.559, "dur": 2.532,
    "args": {
      "External id": 2484385,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759834.483, "dur": 6.026,
    "args": {
      "External id": 2484386,"Record function id": 0, "Sequence number": 24740520, "Fwd thread id": 1, "Ev Idx": 1185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759835.570, "dur": 2.462,
    "args": {
      "External id": 2484387,"Sequence number": 24740520, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1186
    }
  },
  {
    "ph": "f", "id": 128, "pid": 1336757, "tid": 1381170, "ts": 1514656759835.570,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759836.530, "dur": 1.356,
    "args": {
      "External id": 2484388,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759836.968, "dur": 0.809,
    "args": {
      "External id": 2484389,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759843.855, "dur": 93.983,
    "args": {
      "External id": 2484390,"Record function id": 0, "Sequence number": 24740519, "Fwd thread id": 1, "Ev Idx": 1189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759844.630, "dur": 86.669,
    "args": {
      "External id": 2484391,"Sequence number": 24740519, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1190
    }
  },
  {
    "ph": "f", "id": 129, "pid": 1336757, "tid": 1381170, "ts": 1514656759844.630,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759850.123, "dur": 2.265,
    "args": {
      "External id": 2484392,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759850.815, "dur": 1.174,
    "args": {
      "External id": 2484393,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759851.441, "dur": 0.434,
    "args": {
      "External id": 2484394,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759853.096, "dur": 26.710,
    "args": {
      "External id": 2484395,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759880.773, "dur": 7.942,
    "args": {
      "External id": 2484396,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759881.249, "dur": 6.938,
    "args": {
      "External id": 2484397,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759887.254, "dur": 0.820,
    "args": {
      "External id": 2484398,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759890.034, "dur": 5.290,
    "args": {
      "External id": 2484399,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759891.365, "dur": 3.464,
    "args": {
      "External id": 2484400,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759892.135, "dur": 2.628,
    "args": {
      "External id": 2484401,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759895.761, "dur": 34.892,
    "args": {
      "External id": 2484402,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759942.345, "dur": 35.785,
    "args": {
      "External id": 2484403,"Record function id": 0, "Sequence number": 24740518, "Fwd thread id": 1, "Ev Idx": 1202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759943.446, "dur": 8.105,
    "args": {
      "External id": 2484404,"Sequence number": 24740518, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1203
    }
  },
  {
    "ph": "f", "id": 130, "pid": 1336757, "tid": 1381170, "ts": 1514656759943.446,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759947.310, "dur": 4.100,
    "args": {
      "External id": 2484405,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759950.094, "dur": 1.203,
    "args": {
      "External id": 2484406,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759954.520, "dur": 21.297,
    "args": {
      "External id": 2484407,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656759984.033, "dur": 44.210,
    "args": {
      "External id": 2484408,"Record function id": 0, "Sequence number": 24740517, "Fwd thread id": 1, "Ev Idx": 1207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760020.229, "dur": 5.385,
    "args": {
      "External id": 2484409,"Sequence number": 24740517, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1208
    }
  },
  {
    "ph": "f", "id": 131, "pid": 1336757, "tid": 1381170, "ts": 1514656760020.229,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760021.185, "dur": 4.210,
    "args": {
      "External id": 2484410,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760022.338, "dur": 2.287,
    "args": {
      "External id": 2484411,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760023.799, "dur": 0.560,
    "args": {
      "External id": 2484412,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760034.032, "dur": 8.539,
    "args": {
      "External id": 2484413,"Record function id": 0, "Ev Idx": 1212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760035.307, "dur": 6.766,
    "args": {
      "External id": 2484414,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760036.787, "dur": 4.769,
    "args": {
      "External id": 2484415,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760039.838, "dur": 1.593,
    "args": {
      "External id": 2484416,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760045.661, "dur": 11.422,
    "args": {
      "External id": 2484417,"Record function id": 0, "Sequence number": 24740516, "Fwd thread id": 1, "Ev Idx": 1216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760047.043, "dur": 7.715,
    "args": {
      "External id": 2484418,"Sequence number": 24740516, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1217
    }
  },
  {
    "ph": "f", "id": 132, "pid": 1336757, "tid": 1381170, "ts": 1514656760047.043,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760047.895, "dur": 6.708,
    "args": {
      "External id": 2484419,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760053.373, "dur": 1.112,
    "args": {
      "External id": 2484420,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760060.223, "dur": 106.332,
    "args": {
      "External id": 2484421,"Record function id": 0, "Sequence number": 24740515, "Fwd thread id": 1, "Ev Idx": 1220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760061.039, "dur": 94.191,
    "args": {
      "External id": 2484422,"Sequence number": 24740515, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1221
    }
  },
  {
    "ph": "f", "id": 133, "pid": 1336757, "tid": 1381170, "ts": 1514656760061.039,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760066.285, "dur": 2.335,
    "args": {
      "External id": 2484423,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760066.862, "dur": 1.317,
    "args": {
      "External id": 2484424,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760067.659, "dur": 0.405,
    "args": {
      "External id": 2484425,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760069.191, "dur": 36.099,
    "args": {
      "External id": 2484426,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760106.353, "dur": 7.204,
    "args": {
      "External id": 2484427,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760106.877, "dur": 6.006,
    "args": {
      "External id": 2484428,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760109.870, "dur": 2.857,
    "args": {
      "External id": 2484429,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760114.940, "dur": 5.154,
    "args": {
      "External id": 2484430,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760116.176, "dur": 3.468,
    "args": {
      "External id": 2484431,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760119.024, "dur": 0.497,
    "args": {
      "External id": 2484432,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760120.663, "dur": 33.771,
    "args": {
      "External id": 2484433,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760170.959, "dur": 27.916,
    "args": {
      "External id": 2484434,"Record function id": 0, "Sequence number": 24740514, "Fwd thread id": 1, "Ev Idx": 1233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760172.027, "dur": 3.791,
    "args": {
      "External id": 2484435,"Sequence number": 24740514, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1234
    }
  },
  {
    "ph": "f", "id": 134, "pid": 1336757, "tid": 1381170, "ts": 1514656760172.027,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760173.424, "dur": 2.238,
    "args": {
      "External id": 2484436,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760174.297, "dur": 1.144,
    "args": {
      "External id": 2484437,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760178.724, "dur": 17.915,
    "args": {
      "External id": 2484438,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760204.525, "dur": 6.943,
    "args": {
      "External id": 2484439,"Record function id": 0, "Sequence number": 24740513, "Fwd thread id": 1, "Ev Idx": 1238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760205.613, "dur": 4.152,
    "args": {
      "External id": 2484440,"Sequence number": 24740513, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1239
    }
  },
  {
    "ph": "f", "id": 135, "pid": 1336757, "tid": 1381170, "ts": 1514656760205.613,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760206.607, "dur": 2.958,
    "args": {
      "External id": 2484441,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760207.374, "dur": 1.687,
    "args": {
      "External id": 2484442,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760208.490, "dur": 0.447,
    "args": {
      "External id": 2484443,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760217.098, "dur": 5.602,
    "args": {
      "External id": 2484444,"Record function id": 0, "Ev Idx": 1243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760218.542, "dur": 3.700,
    "args": {
      "External id": 2484445,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760219.765, "dur": 2.029,
    "args": {
      "External id": 2484446,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760220.287, "dur": 1.411,
    "args": {
      "External id": 2484447,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760228.757, "dur": 367.120,
    "args": {
      "External id": 2484448,"Record function id": 0, "Sequence number": 24740512, "Fwd thread id": 1, "Ev Idx": 1247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760229.724, "dur": 332.164,
    "args": {
      "External id": 2484449,"Sequence number": 24740512, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1248
    }
  },
  {
    "ph": "f", "id": 136, "pid": 1336757, "tid": 1381170, "ts": 1514656760229.724,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760265.203, "dur": 1.726,
    "args": {
      "External id": 2484450,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760265.742, "dur": 1.050,
    "args": {
      "External id": 2484451,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760280.259, "dur": 6.191,
    "args": {
      "External id": 2484452,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760294.928, "dur": 1.724,
    "args": {
      "External id": 2484453,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760437.440, "dur": 1.588,
    "args": {
      "External id": 2484454,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760442.832, "dur": 34.265,
    "args": {
      "External id": 2484455,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760452.087, "dur": 0.894,
    "args": {
      "External id": 2484456,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760482.240, "dur": 36.056,
    "args": {
      "External id": 2484457,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760486.275, "dur": 31.809,
    "args": {
      "External id": 2484458,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760492.162, "dur": 5.990,
    "args": {
      "External id": 2484459,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760499.364, "dur": 18.023,
    "args": {
      "External id": 2484460,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760522.291, "dur": 2.477,
    "args": {
      "External id": 2484461,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760523.412, "dur": 1.249,
    "args": {
      "External id": 2484462,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760550.091, "dur": 4.639,
    "args": {
      "External id": 2484463,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760552.892, "dur": 1.609,
    "args": {
      "External id": 2484464,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760572.457, "dur": 18.067,
    "args": {
      "External id": 2484465,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760608.354, "dur": 8.578,
    "args": {
      "External id": 2484466,"Record function id": 0, "Ev Idx": 1265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760610.331, "dur": 5.934,
    "args": {
      "External id": 2484467,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760612.312, "dur": 2.972,
    "args": {
      "External id": 2484468,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760613.334, "dur": 1.847,
    "args": {
      "External id": 2484469,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760620.492, "dur": 7.592,
    "args": {
      "External id": 2484470,"Record function id": 0, "Sequence number": 24740511, "Fwd thread id": 1, "Ev Idx": 1269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "AddBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760622.387, "dur": 1.444,
    "args": {
      "External id": 2484471,"Sequence number": 24740511, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1270
    }
  },
  {
    "ph": "f", "id": 137, "pid": 1336757, "tid": 1381170, "ts": 1514656760622.387,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: SwiGLULinearFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760631.771, "dur": 454.652,
    "args": {
      "External id": 2484472,"Record function id": 0, "Sequence number": 24740510, "Fwd thread id": 1, "Ev Idx": 1271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760634.907, "dur": 437.417,
    "args": {
      "External id": 2484473,"Sequence number": 24740510, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1272
    }
  },
  {
    "ph": "f", "id": 138, "pid": 1336757, "tid": 1381170, "ts": 1514656760634.907,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760666.368, "dur": 7.842,
    "args": {
      "External id": 2484474,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760670.499, "dur": 3.449,
    "args": {
      "External id": 2484475,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]", "[8192, 1]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[33554432, 8192, 1], [], []], "Input Dims": [[16, 4096, 2048], [], []], "Ev Idx": 1274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760677.423, "dur": 7.069,
    "args": {
      "External id": 2484476,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760678.885, "dur": 5.018,
    "args": {
      "External id": 2484477,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760680.555, "dur": 3.165,
    "args": {
      "External id": 2484478,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760688.162, "dur": 88.971,
    "args": {
      "External id": 2484479,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8192, 1], [1, 5632], []], "Input Dims": [[65536, 2048], [5632, 2048], []], "Ev Idx": 1278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760691.640, "dur": 4.236,
    "args": {
      "External id": 2484480,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 5632]], "Input Dims": [[5632, 2048]], "Ev Idx": 1279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760692.228, "dur": 3.226,
    "args": {
      "External id": 2484481,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 5632], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760694.767, "dur": 0.594,
    "args": {
      "External id": 2484482,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[5632, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 5632], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760697.254, "dur": 79.330,
    "args": {
      "External id": 2484483,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 1282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760698.603, "dur": 77.148,
    "args": {
      "External id": 2484484,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8192, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 1283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760780.808, "dur": 2.644,
    "args": {
      "External id": 2484485,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [23068672, 5632, 1]], "Input Dims": [[65536, 5632], [16, 4096, 5632]], "Ev Idx": 1284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760781.849, "dur": 1.470,
    "args": {
      "External id": 2484486,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 1285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760815.146, "dur": 6.458,
    "args": {
      "External id": 2484487,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760824.959, "dur": 2.249,
    "args": {
      "External id": 2484488,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760828.088, "dur": 1.809,
    "args": {
      "External id": 2484489,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760863.581, "dur": 2.039,
    "args": {
      "External id": 2484490,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760864.220, "dur": 1.226,
    "args": {
      "External id": 2484491,"Record function id": 0, "Concrete Inputs": ["", "[-1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::einsum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760887.108, "dur": 163.942,
    "args": {
      "External id": 2484492,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["", "TensorList", ""], "Input Strides": [[], [[8192, 1], [5632, 1]], []], "Input Dims": [[], [[65536, 2048], [65536, 5632]], []], "Ev Idx": 1291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760892.009, "dur": 9.127,
    "args": {
      "External id": 2484493,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760896.830, "dur": 3.250,
    "args": {
      "External id": 2484494,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048, 1]", "[8192, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760902.499, "dur": 6.642,
    "args": {
      "External id": 2484495,"Record function id": 0, "Concrete Inputs": ["", "[1, 2, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8192, 1, 1], []], "Input Dims": [[65536, 2048, 1], []], "Ev Idx": 1294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760907.524, "dur": 0.584,
    "args": {
      "External id": 2484496,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 65536]", "[1, 1, 8192]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[65536, 2048, 1], [], [], []], "Ev Idx": 1295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760910.480, "dur": 1.278,
    "args": {
      "External id": 2484497,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 1296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760911.097, "dur": 0.334,
    "args": {
      "External id": 2484498,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760914.296, "dur": 2.121,
    "args": {
      "External id": 2484499,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 1298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760915.192, "dur": 0.678,
    "args": {
      "External id": 2484500,"Record function id": 0, "Concrete Inputs": ["", "[1, 5632, 65536]", "[1, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1, 1], [], [], []], "Input Dims": [[65536, 5632, 1], [], [], []], "Ev Idx": 1299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760919.636, "dur": 3.967,
    "args": {
      "External id": 2484501,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 8192], []], "Input Dims": [[2048, 1, 65536], []], "Ev Idx": 1300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760922.543, "dur": 0.782,
    "args": {
      "External id": 2484502,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536, 1]", "[1, 8192, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 8192], [], [], []], "Input Dims": [[2048, 1, 65536], [], [], []], "Ev Idx": 1301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760924.306, "dur": 6.794,
    "args": {
      "External id": 2484503,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 8192, 1], []], "Input Dims": [[2048, 65536, 1], []], "Ev Idx": 1302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_reshape_alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760929.015, "dur": 1.926,
    "args": {
      "External id": 2484504,"Record function id": 0, "Concrete Inputs": ["", "[1, 2048, 65536]", "[2048, 1, 8192]"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList"], "Input Strides": [[1, 8192, 1], [], []], "Input Dims": [[2048, 65536, 1], [], []], "Ev Idx": 1303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760933.156, "dur": 3.241,
    "args": {
      "External id": 2484505,"Record function id": 0, "Concrete Inputs": ["", "[2, 1, 0]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1, 1, 5632], []], "Input Dims": [[1, 5632, 65536], []], "Ev Idx": 1304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760935.685, "dur": 0.438,
    "args": {
      "External id": 2484506,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632, 1]", "[5632, 1, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1, 5632], [], [], []], "Input Dims": [[1, 5632, 65536], [], [], []], "Ev Idx": 1305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760936.823, "dur": 4.326,
    "args": {
      "External id": 2484507,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 1306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760937.430, "dur": 3.626,
    "args": {
      "External id": 2484508,"Record function id": 0, "Concrete Inputs": ["", "[1, 65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 1], []], "Input Dims": [[65536, 5632, 1], []], "Ev Idx": 1307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656760942.194, "dur": 91.164,
    "args": {
      "External id": 2484509,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1, 8192], [369098752, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632]], "Ev Idx": 1308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761038.251, "dur": 2.023,
    "args": {
      "External id": 2484510,"Record function id": 0, "Concrete Inputs": ["", "[2048, 1, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[11534336, 5632, 1], []], "Input Dims": [[1, 2048, 5632], []], "Ev Idx": 1309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::permute", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761041.268, "dur": 2.929,
    "args": {
      "External id": 2484511,"Record function id": 0, "Concrete Inputs": ["", "[0, 2, 1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 5632, 1], []], "Input Dims": [[2048, 1, 5632], []], "Ev Idx": 1310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761042.538, "dur": 1.019,
    "args": {
      "External id": 2484512,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632, 1]", "[5632, 1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 5632, 1], [], [], []], "Input Dims": [[2048, 1, 5632], [], [], []], "Ev Idx": 1311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761049.147, "dur": 0.822,
    "args": {
      "External id": 2484513,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1, 5632], []], "Input Dims": [[2048, 5632, 1], []], "Ev Idx": 1312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761097.880, "dur": 8.254,
    "args": {
      "External id": 2484514,"Record function id": 0, "Ev Idx": 1313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761099.995, "dur": 5.426,
    "args": {
      "External id": 2484515,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761101.804, "dur": 2.872,
    "args": {
      "External id": 2484516,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761102.602, "dur": 1.933,
    "args": {
      "External id": 2484517,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761109.938, "dur": 7.569,
    "args": {
      "External id": 2484518,"Record function id": 0, "Sequence number": 24740509, "Fwd thread id": 1, "Ev Idx": 1317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761110.875, "dur": 3.785,
    "args": {
      "External id": 2484519,"Sequence number": 24740509, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 1318
    }
  },
  {
    "ph": "f", "id": 139, "pid": 1336757, "tid": 1381170, "ts": 1514656761110.875,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761112.571, "dur": 1.878,
    "args": {
      "External id": 2484520,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761113.467, "dur": 0.848,
    "args": {
      "External id": 2484521,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761120.860, "dur": 132.371,
    "args": {
      "External id": 2484522,"Record function id": 0, "Sequence number": 24740508, "Fwd thread id": 1, "Ev Idx": 1321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761121.863, "dur": 123.874,
    "args": {
      "External id": 2484523,"Sequence number": 24740508, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1322
    }
  },
  {
    "ph": "f", "id": 140, "pid": 1336757, "tid": 1381170, "ts": 1514656761121.863,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761126.218, "dur": 6.210,
    "args": {
      "External id": 2484524,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761127.543, "dur": 4.308,
    "args": {
      "External id": 2484525,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 1324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761130.962, "dur": 0.705,
    "args": {
      "External id": 2484526,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761133.908, "dur": 48.552,
    "args": {
      "External id": 2484527,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 1326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761183.497, "dur": 5.707,
    "args": {
      "External id": 2484528,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761184.189, "dur": 4.020,
    "args": {
      "External id": 2484529,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761186.990, "dur": 1.072,
    "args": {
      "External id": 2484530,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761191.030, "dur": 6.880,
    "args": {
      "External id": 2484531,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761194.303, "dur": 3.200,
    "args": {
      "External id": 2484532,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761195.053, "dur": 2.384,
    "args": {
      "External id": 2484533,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761198.363, "dur": 46.487,
    "args": {
      "External id": 2484534,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761257.980, "dur": 6.046,
    "args": {
      "External id": 2484535,"Record function id": 0, "Sequence number": 24740507, "Fwd thread id": 1, "Ev Idx": 1334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761258.983, "dur": 3.815,
    "args": {
      "External id": 2484536,"Sequence number": 24740507, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1335
    }
  },
  {
    "ph": "f", "id": 141, "pid": 1336757, "tid": 1381170, "ts": 1514656761258.983,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761260.765, "dur": 1.857,
    "args": {
      "External id": 2484537,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761261.453, "dur": 1.034,
    "args": {
      "External id": 2484538,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761267.513, "dur": 11.246,
    "args": {
      "External id": 2484539,"Record function id": 0, "Sequence number": 24740506, "Fwd thread id": 1, "Ev Idx": 1338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761268.405, "dur": 7.688,
    "args": {
      "External id": 2484540,"Sequence number": 24740506, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1339
    }
  },
  {
    "ph": "f", "id": 142, "pid": 1336757, "tid": 1381170, "ts": 1514656761268.405,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761271.274, "dur": 4.583,
    "args": {
      "External id": 2484541,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761271.822, "dur": 3.551,
    "args": {
      "External id": 2484542,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761274.779, "dur": 0.474,
    "args": {
      "External id": 2484543,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761282.888, "dur": 5.567,
    "args": {
      "External id": 2484544,"Record function id": 0, "Ev Idx": 1343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761284.254, "dur": 3.718,
    "args": {
      "External id": 2484545,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761285.546, "dur": 1.955,
    "args": {
      "External id": 2484546,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761286.260, "dur": 1.159,
    "args": {
      "External id": 2484547,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761291.814, "dur": 8.262,
    "args": {
      "External id": 2484548,"Record function id": 0, "Sequence number": 24740505, "Fwd thread id": 1, "Ev Idx": 1347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761292.603, "dur": 4.767,
    "args": {
      "External id": 2484549,"Sequence number": 24740505, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[23068672, 5632, 1]], "Input Dims": [[16, 4096, 5632]], "Ev Idx": 1348
    }
  },
  {
    "ph": "f", "id": 143, "pid": 1336757, "tid": 1381170, "ts": 1514656761292.603,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761293.773, "dur": 3.465,
    "args": {
      "External id": 2484550,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761296.358, "dur": 0.746,
    "args": {
      "External id": 2484551,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 1350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761303.357, "dur": 116.532,
    "args": {
      "External id": 2484552,"Record function id": 0, "Sequence number": 24740504, "Fwd thread id": 1, "Ev Idx": 1351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761304.214, "dur": 106.374,
    "args": {
      "External id": 2484553,"Sequence number": 24740504, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1352
    }
  },
  {
    "ph": "f", "id": 144, "pid": 1336757, "tid": 1381170, "ts": 1514656761304.214,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761307.496, "dur": 4.380,
    "args": {
      "External id": 2484554,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[65536, 5632]], "Ev Idx": 1353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761307.917, "dur": 3.535,
    "args": {
      "External id": 2484555,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[65536, 5632], [], []], "Ev Idx": 1354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761310.964, "dur": 0.388,
    "args": {
      "External id": 2484556,"Record function id": 0, "Concrete Inputs": ["", "[5632, 65536]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[65536, 5632], [], [], []], "Ev Idx": 1355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761312.637, "dur": 34.406,
    "args": {
      "External id": 2484557,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048]], "Ev Idx": 1356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761350.229, "dur": 5.764,
    "args": {
      "External id": 2484558,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761350.686, "dur": 4.732,
    "args": {
      "External id": 2484559,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 1358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761351.604, "dur": 3.659,
    "args": {
      "External id": 2484560,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 1359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761357.343, "dur": 3.436,
    "args": {
      "External id": 2484561,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761358.853, "dur": 1.539,
    "args": {
      "External id": 2484562,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761359.709, "dur": 0.555,
    "args": {
      "External id": 2484563,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761363.181, "dur": 46.725,
    "args": {
      "External id": 2484564,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 1363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761426.582, "dur": 33.859,
    "args": {
      "External id": 2484565,"Record function id": 0, "Sequence number": 24740503, "Fwd thread id": 1, "Ev Idx": 1364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761427.481, "dur": 5.225,
    "args": {
      "External id": 2484566,"Sequence number": 24740503, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1365
    }
  },
  {
    "ph": "f", "id": 145, "pid": 1336757, "tid": 1381170, "ts": 1514656761427.481,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761430.693, "dur": 1.855,
    "args": {
      "External id": 2484567,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761431.453, "dur": 0.961,
    "args": {
      "External id": 2484568,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761435.680, "dur": 21.602,
    "args": {
      "External id": 2484569,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761464.769, "dur": 9.707,
    "args": {
      "External id": 2484570,"Record function id": 0, "Sequence number": 24740502, "Fwd thread id": 1, "Ev Idx": 1369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761465.809, "dur": 6.461,
    "args": {
      "External id": 2484571,"Sequence number": 24740502, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1370
    }
  },
  {
    "ph": "f", "id": 146, "pid": 1336757, "tid": 1381170, "ts": 1514656761465.809,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761466.524, "dur": 5.538,
    "args": {
      "External id": 2484572,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 5632]], "Ev Idx": 1371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761467.422, "dur": 4.108,
    "args": {
      "External id": 2484573,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 1372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761470.549, "dur": 0.870,
    "args": {
      "External id": 2484574,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 1373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761478.362, "dur": 7.336,
    "args": {
      "External id": 2484575,"Record function id": 0, "Ev Idx": 1374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761481.829, "dur": 3.346,
    "args": {
      "External id": 2484576,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761482.841, "dur": 1.845,
    "args": {
      "External id": 2484577,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761483.557, "dur": 1.017,
    "args": {
      "External id": 2484578,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761489.643, "dur": 446.403,
    "args": {
      "External id": 2484579,"Record function id": 0, "Sequence number": 24740501, "Fwd thread id": 1, "Ev Idx": 1378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761494.447, "dur": 404.410,
    "args": {
      "External id": 2484580,"Sequence number": 24740501, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 1379
    }
  },
  {
    "ph": "f", "id": 147, "pid": 1336757, "tid": 1381170, "ts": 1514656761494.447,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761515.488, "dur": 50.173,
    "args": {
      "External id": 2484581,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761516.990, "dur": 48.435,
    "args": {
      "External id": 2484582,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761519.564, "dur": 5.418,
    "args": {
      "External id": 2484583,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], [], []], "Ev Idx": 1382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761521.891, "dur": 2.624,
    "args": {
      "External id": 2484584,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761526.147, "dur": 38.328,
    "args": {
      "External id": 2484585,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761581.344, "dur": 6.448,
    "args": {
      "External id": 2484586,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761584.002, "dur": 3.678,
    "args": {
      "External id": 2484587,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761591.384, "dur": 1.440,
    "args": {
      "External id": 2484588,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761591.991, "dur": 0.727,
    "args": {
      "External id": 2484589,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761607.399, "dur": 2.886,
    "args": {
      "External id": 2484590,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761621.083, "dur": 3.400,
    "args": {
      "External id": 2484591,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761784.461, "dur": 3.934,
    "args": {
      "External id": 2484592,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761791.931, "dur": 33.924,
    "args": {
      "External id": 2484593,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761802.703, "dur": 0.851,
    "args": {
      "External id": 2484594,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761831.341, "dur": 31.492,
    "args": {
      "External id": 2484595,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761833.105, "dur": 29.511,
    "args": {
      "External id": 2484596,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761836.540, "dur": 3.563,
    "args": {
      "External id": 2484597,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761843.202, "dur": 18.851,
    "args": {
      "External id": 2484598,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761867.157, "dur": 5.026,
    "args": {
      "External id": 2484599,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761870.869, "dur": 1.177,
    "args": {
      "External id": 2484600,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761878.177, "dur": 4.199,
    "args": {
      "External id": 2484601,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761878.766, "dur": 3.491,
    "args": {
      "External id": 2484602,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761884.133, "dur": 4.344,
    "args": {
      "External id": 2484603,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761887.075, "dur": 1.321,
    "args": {
      "External id": 2484604,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761915.725, "dur": 18.691,
    "args": {
      "External id": 2484605,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761948.181, "dur": 8.407,
    "args": {
      "External id": 2484606,"Record function id": 0, "Ev Idx": 1405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761950.458, "dur": 5.488,
    "args": {
      "External id": 2484607,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761952.234, "dur": 2.843,
    "args": {
      "External id": 2484608,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761952.999, "dur": 1.980,
    "args": {
      "External id": 2484609,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761960.010, "dur": 9.205,
    "args": {
      "External id": 2484610,"Record function id": 0, "Sequence number": 24740500, "Fwd thread id": 1, "Ev Idx": 1409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761960.934, "dur": 5.497,
    "args": {
      "External id": 2484611,"Sequence number": 24740500, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1410
    }
  },
  {
    "ph": "f", "id": 148, "pid": 1336757, "tid": 1381170, "ts": 1514656761960.934,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761962.418, "dur": 3.828,
    "args": {
      "External id": 2484612,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761965.137, "dur": 0.975,
    "args": {
      "External id": 2484613,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761972.761, "dur": 188.532,
    "args": {
      "External id": 2484614,"Record function id": 0, "Sequence number": 24740499, "Fwd thread id": 1, "Ev Idx": 1413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761973.630, "dur": 178.363,
    "args": {
      "External id": 2484615,"Sequence number": 24740499, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1414
    }
  },
  {
    "ph": "f", "id": 149, "pid": 1336757, "tid": 1381170, "ts": 1514656761973.630,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761976.975, "dur": 3.870,
    "args": {
      "External id": 2484616,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761978.032, "dur": 2.306,
    "args": {
      "External id": 2484617,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761979.139, "dur": 0.993,
    "args": {
      "External id": 2484618,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656761981.812, "dur": 105.513,
    "args": {
      "External id": 2484619,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762092.163, "dur": 9.843,
    "args": {
      "External id": 2484620,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762093.085, "dur": 7.981,
    "args": {
      "External id": 2484621,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762097.974, "dur": 2.928,
    "args": {
      "External id": 2484622,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762104.017, "dur": 2.918,
    "args": {
      "External id": 2484623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762105.189, "dur": 1.257,
    "args": {
      "External id": 2484624,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762105.927, "dur": 0.450,
    "args": {
      "External id": 2484625,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762109.784, "dur": 41.527,
    "args": {
      "External id": 2484626,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762168.049, "dur": 12.153,
    "args": {
      "External id": 2484627,"Record function id": 0, "Sequence number": 24740498, "Fwd thread id": 1, "Ev Idx": 1426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762169.112, "dur": 8.450,
    "args": {
      "External id": 2484628,"Sequence number": 24740498, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1427
    }
  },
  {
    "ph": "f", "id": 150, "pid": 1336757, "tid": 1381170, "ts": 1514656762169.112,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762170.643, "dur": 6.752,
    "args": {
      "External id": 2484629,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762173.631, "dur": 3.665,
    "args": {
      "External id": 2484630,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762183.472, "dur": 9.253,
    "args": {
      "External id": 2484631,"Record function id": 0, "Sequence number": 24740497, "Fwd thread id": 1, "Ev Idx": 1430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762184.439, "dur": 6.360,
    "args": {
      "External id": 2484632,"Sequence number": 24740497, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1431
    }
  },
  {
    "ph": "f", "id": 151, "pid": 1336757, "tid": 1381170, "ts": 1514656762184.439,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762185.541, "dur": 5.024,
    "args": {
      "External id": 2484633,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762186.211, "dur": 3.908,
    "args": {
      "External id": 2484634,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762189.628, "dur": 0.389,
    "args": {
      "External id": 2484635,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762196.851, "dur": 5.353,
    "args": {
      "External id": 2484636,"Record function id": 0, "Ev Idx": 1435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762198.093, "dur": 3.594,
    "args": {
      "External id": 2484637,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762199.375, "dur": 2.030,
    "args": {
      "External id": 2484638,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762199.908, "dur": 1.403,
    "args": {
      "External id": 2484639,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762205.394, "dur": 6.819,
    "args": {
      "External id": 2484640,"Record function id": 0, "Sequence number": 24740496, "Fwd thread id": 1, "Ev Idx": 1439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762206.128, "dur": 3.419,
    "args": {
      "External id": 2484641,"Sequence number": 24740496, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1440
    }
  },
  {
    "ph": "f", "id": 152, "pid": 1336757, "tid": 1381170, "ts": 1514656762206.128,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762207.447, "dur": 1.941,
    "args": {
      "External id": 2484642,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762208.037, "dur": 1.230,
    "args": {
      "External id": 2484643,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: FlashAttnFuncBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762216.353, "dur": 292.317,
    "args": {
      "External id": 2484644,"Record function id": 0, "Sequence number": 24740495, "Fwd thread id": 1, "Ev Idx": 1443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFuncBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762219.754, "dur": 266.174,
    "args": {
      "External id": 2484645,"Sequence number": 24740495, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1444
    }
  },
  {
    "ph": "f", "id": 153, "pid": 1336757, "tid": 1381170, "ts": 1514656762219.754,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762236.804, "dur": 8.615,
    "args": {
      "External id": 2484646,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762241.156, "dur": 3.857,
    "args": {
      "External id": 2484647,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762247.324, "dur": 5.411,
    "args": {
      "External id": 2484648,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762250.191, "dur": 2.314,
    "args": {
      "External id": 2484649,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762254.083, "dur": 2.660,
    "args": {
      "External id": 2484650,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762254.784, "dur": 1.752,
    "args": {
      "External id": 2484651,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762281.289, "dur": 180.735,
    "args": {
      "External id": 2484652,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762362.871, "dur": 5.924,
    "args": {
      "External id": 2484653,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762372.228, "dur": 4.056,
    "args": {
      "External id": 2484654,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762473.420, "dur": 3.642,
    "args": {
      "External id": 2484655,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762479.767, "dur": 0.621,
    "args": {
      "External id": 2484656,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762482.271, "dur": 0.892,
    "args": {
      "External id": 2484657,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762515.377, "dur": 257.647,
    "args": {
      "External id": 2484658,"Record function id": 0, "Sequence number": 24740494, "Fwd thread id": 1, "Ev Idx": 1457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762516.831, "dur": 247.305,
    "args": {
      "External id": 2484659,"Sequence number": 24740494, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1458
    }
  },
  {
    "ph": "f", "id": 154, "pid": 1336757, "tid": 1381170, "ts": 1514656762516.831,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762555.425, "dur": 54.813,
    "args": {
      "External id": 2484660,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762560.289, "dur": 4.051,
    "args": {
      "External id": 2484661,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762565.956, "dur": 43.693,
    "args": {
      "External id": 2484662,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762620.387, "dur": 4.302,
    "args": {
      "External id": 2484663,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762621.853, "dur": 2.501,
    "args": {
      "External id": 2484664,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762781.512, "dur": 155.734,
    "args": {
      "External id": 2484665,"Record function id": 0, "Sequence number": 24740493, "Fwd thread id": 1, "Ev Idx": 1464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762785.601, "dur": 144.113,
    "args": {
      "External id": 2484666,"Sequence number": 24740493, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1465
    }
  },
  {
    "ph": "f", "id": 155, "pid": 1336757, "tid": 1381170, "ts": 1514656762785.601,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762797.660, "dur": 32.070,
    "args": {
      "External id": 2484667,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762799.451, "dur": 2.713,
    "args": {
      "External id": 2484668,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762803.216, "dur": 25.947,
    "args": {
      "External id": 2484669,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], []], "Ev Idx": 1468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762836.605, "dur": 3.882,
    "args": {
      "External id": 2484670,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 1469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762837.945, "dur": 2.247,
    "args": {
      "External id": 2484671,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762943.094, "dur": 16.215,
    "args": {
      "External id": 2484672,"Record function id": 0, "Sequence number": 24740492, "Fwd thread id": 1, "Ev Idx": 1471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762944.242, "dur": 12.008,
    "args": {
      "External id": 2484673,"Sequence number": 24740492, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1472
    }
  },
  {
    "ph": "f", "id": 156, "pid": 1336757, "tid": 1381170, "ts": 1514656762944.242,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762948.781, "dur": 7.202,
    "args": {
      "External id": 2484674,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762949.850, "dur": 5.957,
    "args": {
      "External id": 2484675,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762962.634, "dur": 6.469,
    "args": {
      "External id": 2484676,"Record function id": 0, "Sequence number": 24740491, "Fwd thread id": 1, "Ev Idx": 1475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762963.690, "dur": 3.597,
    "args": {
      "External id": 2484677,"Sequence number": 24740491, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1476
    }
  },
  {
    "ph": "f", "id": 157, "pid": 1336757, "tid": 1381170, "ts": 1514656762963.690,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762964.986, "dur": 2.157,
    "args": {
      "External id": 2484678,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762965.823, "dur": 1.191,
    "args": {
      "External id": 2484679,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762972.356, "dur": 9.307,
    "args": {
      "External id": 2484680,"Record function id": 0, "Sequence number": 24740490, "Fwd thread id": 1, "Ev Idx": 1479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762973.414, "dur": 6.369,
    "args": {
      "External id": 2484681,"Sequence number": 24740490, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 1480
    }
  },
  {
    "ph": "f", "id": 158, "pid": 1336757, "tid": 1381170, "ts": 1514656762973.414,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762978.116, "dur": 1.527,
    "args": {
      "External id": 2484682,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656762978.702, "dur": 0.853,
    "args": {
      "External id": 2484683,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 1482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763021.240, "dur": 8.721,
    "args": {
      "External id": 2484684,"Record function id": 0, "Sequence number": 24740489, "Fwd thread id": 1, "Ev Idx": 1483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763023.173, "dur": 4.232,
    "args": {
      "External id": 2484685,"Sequence number": 24740489, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1484
    }
  },
  {
    "ph": "f", "id": 159, "pid": 1336757, "tid": 1381170, "ts": 1514656763023.173,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763024.594, "dur": 2.664,
    "args": {
      "External id": 2484686,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763025.267, "dur": 1.765,
    "args": {
      "External id": 2484687,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763034.383, "dur": 153.592,
    "args": {
      "External id": 2484688,"Record function id": 0, "Sequence number": 24740488, "Fwd thread id": 1, "Ev Idx": 1487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763035.655, "dur": 141.878,
    "args": {
      "External id": 2484689,"Sequence number": 24740488, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1488
    }
  },
  {
    "ph": "f", "id": 160, "pid": 1336757, "tid": 1381170, "ts": 1514656763035.655,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763040.092, "dur": 10.391,
    "args": {
      "External id": 2484690,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763043.761, "dur": 5.993,
    "args": {
      "External id": 2484691,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763047.827, "dur": 1.660,
    "args": {
      "External id": 2484692,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763051.907, "dur": 66.851,
    "args": {
      "External id": 2484693,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763120.138, "dur": 4.270,
    "args": {
      "External id": 2484694,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763120.781, "dur": 2.555,
    "args": {
      "External id": 2484695,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763122.037, "dur": 1.174,
    "args": {
      "External id": 2484696,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763126.248, "dur": 6.227,
    "args": {
      "External id": 2484697,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763130.641, "dur": 1.379,
    "args": {
      "External id": 2484698,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763131.322, "dur": 0.573,
    "args": {
      "External id": 2484699,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763133.146, "dur": 43.638,
    "args": {
      "External id": 2484700,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763192.896, "dur": 8.755,
    "args": {
      "External id": 2484701,"Record function id": 0, "Sequence number": 24740487, "Fwd thread id": 1, "Ev Idx": 1500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763193.931, "dur": 5.204,
    "args": {
      "External id": 2484702,"Sequence number": 24740487, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1501
    }
  },
  {
    "ph": "f", "id": 161, "pid": 1336757, "tid": 1381170, "ts": 1514656763193.931,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763197.241, "dur": 1.757,
    "args": {
      "External id": 2484703,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763197.792, "dur": 1.099,
    "args": {
      "External id": 2484704,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763205.308, "dur": 8.530,
    "args": {
      "External id": 2484705,"Record function id": 0, "Sequence number": 24740486, "Fwd thread id": 1, "Ev Idx": 1504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763206.065, "dur": 5.026,
    "args": {
      "External id": 2484706,"Sequence number": 24740486, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1505
    }
  },
  {
    "ph": "f", "id": 162, "pid": 1336757, "tid": 1381170, "ts": 1514656763206.065,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763208.534, "dur": 2.352,
    "args": {
      "External id": 2484707,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763209.110, "dur": 1.312,
    "args": {
      "External id": 2484708,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763209.911, "dur": 0.369,
    "args": {
      "External id": 2484709,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763219.493, "dur": 8.483,
    "args": {
      "External id": 2484710,"Record function id": 0, "Ev Idx": 1509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763220.952, "dur": 6.298,
    "args": {
      "External id": 2484711,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763223.380, "dur": 3.457,
    "args": {
      "External id": 2484712,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763224.230, "dur": 2.460,
    "args": {
      "External id": 2484713,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763231.350, "dur": 10.472,
    "args": {
      "External id": 2484714,"Record function id": 0, "Sequence number": 24740485, "Fwd thread id": 1, "Ev Idx": 1513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763234.895, "dur": 4.108,
    "args": {
      "External id": 2484715,"Sequence number": 24740485, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1514
    }
  },
  {
    "ph": "f", "id": 163, "pid": 1336757, "tid": 1381170, "ts": 1514656763234.895,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763235.795, "dur": 3.049,
    "args": {
      "External id": 2484716,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763237.820, "dur": 0.883,
    "args": {
      "External id": 2484717,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763244.759, "dur": 94.642,
    "args": {
      "External id": 2484718,"Record function id": 0, "Sequence number": 24740484, "Fwd thread id": 1, "Ev Idx": 1517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763245.543, "dur": 87.482,
    "args": {
      "External id": 2484719,"Sequence number": 24740484, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1518
    }
  },
  {
    "ph": "f", "id": 164, "pid": 1336757, "tid": 1381170, "ts": 1514656763245.543,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763248.836, "dur": 2.154,
    "args": {
      "External id": 2484720,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763249.331, "dur": 1.233,
    "args": {
      "External id": 2484721,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763250.035, "dur": 0.388,
    "args": {
      "External id": 2484722,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763251.798, "dur": 28.410,
    "args": {
      "External id": 2484723,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763283.414, "dur": 3.499,
    "args": {
      "External id": 2484724,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763284.086, "dur": 2.282,
    "args": {
      "External id": 2484725,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763284.944, "dur": 1.304,
    "args": {
      "External id": 2484726,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763288.360, "dur": 7.960,
    "args": {
      "External id": 2484727,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763289.717, "dur": 6.083,
    "args": {
      "External id": 2484728,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763292.901, "dur": 2.827,
    "args": {
      "External id": 2484729,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763298.948, "dur": 33.381,
    "args": {
      "External id": 2484730,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763345.808, "dur": 35.708,
    "args": {
      "External id": 2484731,"Record function id": 0, "Sequence number": 24740483, "Fwd thread id": 1, "Ev Idx": 1530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763346.810, "dur": 7.620,
    "args": {
      "External id": 2484732,"Sequence number": 24740483, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1531
    }
  },
  {
    "ph": "f", "id": 165, "pid": 1336757, "tid": 1381170, "ts": 1514656763346.810,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763348.366, "dur": 5.918,
    "args": {
      "External id": 2484733,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763353.037, "dur": 1.137,
    "args": {
      "External id": 2484734,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763357.461, "dur": 21.843,
    "args": {
      "External id": 2484735,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763385.623, "dur": 34.562,
    "args": {
      "External id": 2484736,"Record function id": 0, "Sequence number": 24740482, "Fwd thread id": 1, "Ev Idx": 1535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763410.129, "dur": 7.856,
    "args": {
      "External id": 2484737,"Sequence number": 24740482, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1536
    }
  },
  {
    "ph": "f", "id": 166, "pid": 1336757, "tid": 1381170, "ts": 1514656763410.129,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763413.170, "dur": 4.618,
    "args": {
      "External id": 2484738,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763413.929, "dur": 3.319,
    "args": {
      "External id": 2484739,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763416.685, "dur": 0.443,
    "args": {
      "External id": 2484740,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763424.135, "dur": 5.222,
    "args": {
      "External id": 2484741,"Record function id": 0, "Ev Idx": 1540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763425.481, "dur": 3.336,
    "args": {
      "External id": 2484742,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763426.612, "dur": 1.782,
    "args": {
      "External id": 2484743,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763426.987, "dur": 1.320,
    "args": {
      "External id": 2484744,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763432.552, "dur": 7.245,
    "args": {
      "External id": 2484745,"Record function id": 0, "Sequence number": 24740481, "Fwd thread id": 1, "Ev Idx": 1544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "UnsafeViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763433.878, "dur": 4.376,
    "args": {
      "External id": 2484746,"Sequence number": 24740481, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1545
    }
  },
  {
    "ph": "f", "id": 167, "pid": 1336757, "tid": 1381170, "ts": 1514656763433.878,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763434.799, "dur": 3.293,
    "args": {
      "External id": 2484747,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763437.029, "dur": 0.951,
    "args": {
      "External id": 2484748,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763442.987, "dur": 116.306,
    "args": {
      "External id": 2484749,"Record function id": 0, "Sequence number": 24740480, "Fwd thread id": 1, "Ev Idx": 1548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "MmBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763443.719, "dur": 105.256,
    "args": {
      "External id": 2484750,"Sequence number": 24740480, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1549
    }
  },
  {
    "ph": "f", "id": 168, "pid": 1336757, "tid": 1381170, "ts": 1514656763443.719,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763446.181, "dur": 4.459,
    "args": {
      "External id": 2484751,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763446.658, "dur": 3.581,
    "args": {
      "External id": 2484752,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[65536, 2048], [], []], "Ev Idx": 1551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763449.702, "dur": 0.422,
    "args": {
      "External id": 2484753,"Record function id": 0, "Concrete Inputs": ["", "[2048, 65536]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 1552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763451.329, "dur": 29.762,
    "args": {
      "External id": 2484754,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048]], "Ev Idx": 1553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763484.683, "dur": 2.633,
    "args": {
      "External id": 2484755,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763485.282, "dur": 1.514,
    "args": {
      "External id": 2484756,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763486.095, "dur": 0.577,
    "args": {
      "External id": 2484757,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763488.744, "dur": 2.675,
    "args": {
      "External id": 2484758,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763489.875, "dur": 1.144,
    "args": {
      "External id": 2484759,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763490.391, "dur": 0.505,
    "args": {
      "External id": 2484760,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763493.900, "dur": 53.759,
    "args": {
      "External id": 2484761,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 1560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763566.072, "dur": 32.047,
    "args": {
      "External id": 2484762,"Record function id": 0, "Sequence number": 24740479, "Fwd thread id": 1, "Ev Idx": 1561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "ViewBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763567.211, "dur": 6.160,
    "args": {
      "External id": 2484763,"Sequence number": 24740479, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 1562
    }
  },
  {
    "ph": "f", "id": 169, "pid": 1336757, "tid": 1381170, "ts": 1514656763567.211,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763570.790, "dur": 2.416,
    "args": {
      "External id": 2484764,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763571.353, "dur": 1.750,
    "args": {
      "External id": 2484765,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763575.900, "dur": 19.866,
    "args": {
      "External id": 2484766,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763601.783, "dur": 9.437,
    "args": {
      "External id": 2484767,"Record function id": 0, "Sequence number": 24740478, "Fwd thread id": 1, "Ev Idx": 1566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TBackward0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763602.630, "dur": 6.924,
    "args": {
      "External id": 2484768,"Sequence number": 24740478, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1567
    }
  },
  {
    "ph": "f", "id": 170, "pid": 1336757, "tid": 1381170, "ts": 1514656763602.630,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763603.515, "dur": 5.811,
    "args": {
      "External id": 2484769,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1, 2048]], "Input Dims": [[2048, 2048]], "Ev Idx": 1568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763604.370, "dur": 4.448,
    "args": {
      "External id": 2484770,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[1, 2048], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 1569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763607.920, "dur": 0.738,
    "args": {
      "External id": 2484771,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[2048, 1]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 2048], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 1570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763615.558, "dur": 5.461,
    "args": {
      "External id": 2484772,"Record function id": 0, "Ev Idx": 1571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763616.945, "dur": 3.604,
    "args": {
      "External id": 2484773,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763618.190, "dur": 2.009,
    "args": {
      "External id": 2484774,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763618.751, "dur": 1.355,
    "args": {
      "External id": 2484775,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763624.892, "dur": 397.891,
    "args": {
      "External id": 2484776,"Record function id": 0, "Sequence number": 24740477, "Fwd thread id": 1, "Ev Idx": 1575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763628.056, "dur": 316.218,
    "args": {
      "External id": 2484777,"Sequence number": 24740477, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1576
    }
  },
  {
    "ph": "f", "id": 171, "pid": 1336757, "tid": 1381170, "ts": 1514656763628.056,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763660.996, "dur": 1.996,
    "args": {
      "External id": 2484778,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763661.657, "dur": 1.171,
    "args": {
      "External id": 2484779,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 1578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763678.706, "dur": 6.003,
    "args": {
      "External id": 2484780,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763693.735, "dur": 2.115,
    "args": {
      "External id": 2484781,"Record function id": 0, "Concrete Inputs": ["[132, 2048]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763840.714, "dur": 1.635,
    "args": {
      "External id": 2484782,"Record function id": 0, "Concrete Inputs": ["", "[1, -1, 2048]"], "Input type": ["float", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[132, 2048], []], "Ev Idx": 1581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763846.416, "dur": 36.497,
    "args": {
      "External id": 2484783,"Record function id": 0, "Concrete Inputs": ["", "[1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[270336, 2048, 1], [], [], []], "Input Dims": [[1, 132, 2048], [], [], []], "Ev Idx": 1582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763858.499, "dur": 0.668,
    "args": {
      "External id": 2484784,"Record function id": 0, "Concrete Inputs": ["", "[1, 1, 2048]", "[2048, 0, 1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[1, 2048], [], [], []], "Ev Idx": 1583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763888.716, "dur": 31.708,
    "args": {
      "External id": 2484785,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], []], "Ev Idx": 1584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763890.464, "dur": 29.741,
    "args": {
      "External id": 2484786,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[1, 2048], [], [], [], [], [], []], "Ev Idx": 1585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763893.971, "dur": 4.847,
    "args": {
      "External id": 2484787,"Record function id": 0, "Concrete Inputs": ["[1, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763902.493, "dur": 17.191,
    "args": {
      "External id": 2484788,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[1, 2048], [1, 2048], []], "Ev Idx": 1587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763925.315, "dur": 4.841,
    "args": {
      "External id": 2484789,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1]], "Input Dims": [[1, 2048], [2048]], "Ev Idx": 1588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763929.062, "dur": 0.997,
    "args": {
      "External id": 2484790,"Record function id": 0, "Concrete Inputs": ["", "[2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[1, 2048], []], "Ev Idx": 1589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763935.905, "dur": 1.644,
    "args": {
      "External id": 2484791,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763936.504, "dur": 0.913,
    "args": {
      "External id": 2484792,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 1591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656763959.239, "dur": 23.244,
    "args": {
      "External id": 2484793,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 1592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764035.019, "dur": 11.504,
    "args": {
      "External id": 2484794,"Record function id": 0, "Ev Idx": 1593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764037.117, "dur": 8.661,
    "args": {
      "External id": 2484795,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764039.122, "dur": 5.577,
    "args": {
      "External id": 2484796,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764042.367, "dur": 2.234,
    "args": {
      "External id": 2484797,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764050.999, "dur": 2860.087,
    "args": {
      "External id": 2484798,"Record function id": 0, "Ev Idx": 1597
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.27)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764086.442, "dur": 1030.436,
    "args": {
      "External id": 2484799,"Record function id": 0, "Ev Idx": 1598
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.26", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764115.503, "dur": 992.141,
    "args": {
      "External id": 2484800,"Record function id": 0, "Ev Idx": 1599
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.26)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764129.870, "dur": 961.331,
    "args": {
      "External id": 2484801,"Record function id": 0, "Ev Idx": 1600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764227.241, "dur": 7.926,
    "args": {
      "External id": 2484802,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764253.464, "dur": 42.427,
    "args": {
      "External id": 2484803,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764260.018, "dur": 3.728,
    "args": {
      "External id": 2484804,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764265.182, "dur": 1.102,
    "args": {
      "External id": 2484805,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764271.549, "dur": 0.377,
    "args": {
      "External id": 2484806,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764272.511, "dur": 0.335,
    "args": {
      "External id": 2484807,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764273.365, "dur": 0.431,
    "args": {
      "External id": 2484808,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764278.822, "dur": 0.525,
    "args": {
      "External id": 2484809,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764280.044, "dur": 0.678,
    "args": {
      "External id": 2484810,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764281.415, "dur": 2.220,
    "args": {
      "External id": 2484811,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764286.228, "dur": 2.530,
    "args": {
      "External id": 2484812,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764313.068, "dur": 41.040,
    "args": {
      "External id": 2484813,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764398.183, "dur": 120.073,
    "args": {
      "External id": 2484814,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 1613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764412.983, "dur": 6.476,
    "args": {
      "External id": 2484815,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764426.514, "dur": 9.393,
    "args": {
      "External id": 2484816,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764430.501, "dur": 4.953,
    "args": {
      "External id": 2484817,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 1616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764433.440, "dur": 0.567,
    "args": {
      "External id": 2484818,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764442.677, "dur": 34.132,
    "args": {
      "External id": 2484819,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764445.607, "dur": 0.337,
    "args": {
      "External id": 2484820,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764447.181, "dur": 2.666,
    "args": {
      "External id": 2484821,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764450.441, "dur": 0.545,
    "args": {
      "External id": 2484822,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764454.438, "dur": 0.526,
    "args": {
      "External id": 2484823,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764457.604, "dur": 0.382,
    "args": {
      "External id": 2484824,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764458.920, "dur": 2.650,
    "args": {
      "External id": 2484825,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764463.734, "dur": 0.386,
    "args": {
      "External id": 2484826,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764467.163, "dur": 0.345,
    "args": {
      "External id": 2484827,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764470.341, "dur": 0.300,
    "args": {
      "External id": 2484828,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764487.396, "dur": 22.733,
    "args": {
      "External id": 2484829,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764603.277, "dur": 330.445,
    "args": {
      "External id": 2484830,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 1629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764643.186, "dur": 285.172,
    "args": {
      "External id": 2484831,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1630, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764654.995, "dur": 266.772,
    "args": {
      "External id": 2484832,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 1631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514656764960.673, "dur": 2.342,
    "args": {
      "External id": 2484833,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1632, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656765125.325, "dur": 1761.279,
    "args": {
      "External id": 2484834,"Sequence number": 24740476, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1633
    }
  },
  {
    "ph": "f", "id": 172, "pid": 1336757, "tid": 1381170, "ts": 1514656765125.325,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656765260.422, "dur": 105.704,
    "args": {
      "External id": 2484835,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 1634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656765412.720, "dur": 42.081,
    "args": {
      "External id": 2484836,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 1635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656765473.034, "dur": 47.130,
    "args": {
      "External id": 2484837,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 1636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656765547.584, "dur": 37.343,
    "args": {
      "External id": 2484838,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656765592.560, "dur": 45.199,
    "args": {
      "External id": 2484839,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656765647.231, "dur": 29.814,
    "args": {
      "External id": 2484840,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656765685.455, "dur": 43.777,
    "args": {
      "External id": 2484841,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514656765759.088, "dur": 27.733,
    "args": {
      "External id": 2484842,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 1641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656765808.206, "dur": 34.892,
    "args": {
      "External id": 2484843,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514656765872.009, "dur": 21.832,
    "args": {
      "External id": 2484844,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514656765910.637, "dur": 20.921,
    "args": {
      "External id": 2484845,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656765941.484, "dur": 30.992,
    "args": {
      "External id": 2484846,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656765976.189, "dur": 86.334,
    "args": {
      "External id": 2484847,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766099.390, "dur": 183.740,
    "args": {
      "External id": 2484848,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766186.408, "dur": 8.243,
    "args": {
      "External id": 2484849,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766196.512, "dur": 2.553,
    "args": {
      "External id": 2484850,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766321.601, "dur": 29.541,
    "args": {
      "External id": 2484851,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766369.095, "dur": 16.060,
    "args": {
      "External id": 2484852,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766393.911, "dur": 37.665,
    "args": {
      "External id": 2484853,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766436.918, "dur": 33.948,
    "args": {
      "External id": 2484854,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766477.534, "dur": 22.059,
    "args": {
      "External id": 2484855,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766506.308, "dur": 47.041,
    "args": {
      "External id": 2484856,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766562.139, "dur": 23.840,
    "args": {
      "External id": 2484857,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766593.476, "dur": 30.555,
    "args": {
      "External id": 2484858,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766647.496, "dur": 25.714,
    "args": {
      "External id": 2484859,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 1658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766696.013, "dur": 39.115,
    "args": {
      "External id": 2484860,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766763.449, "dur": 21.152,
    "args": {
      "External id": 2484861,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766798.952, "dur": 17.229,
    "args": {
      "External id": 2484862,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766838.663, "dur": 18.876,
    "args": {
      "External id": 2484863,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 1662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766933.747, "dur": 13.612,
    "args": {
      "External id": 2484864,"Record function id": 0, "Ev Idx": 1663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766936.563, "dur": 9.939,
    "args": {
      "External id": 2484865,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766940.322, "dur": 5.213,
    "args": {
      "External id": 2484866,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766941.660, "dur": 3.768,
    "args": {
      "External id": 2484867,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766950.988, "dur": 6.592,
    "args": {
      "External id": 2484868,"Record function id": 0, "Ev Idx": 1667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766954.474, "dur": 2.585,
    "args": {
      "External id": 2484869,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766955.166, "dur": 1.368,
    "args": {
      "External id": 2484870,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766955.587, "dur": 0.883,
    "args": {
      "External id": 2484871,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766960.927, "dur": 4.567,
    "args": {
      "External id": 2484872,"Record function id": 0, "Ev Idx": 1671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766962.337, "dur": 2.698,
    "args": {
      "External id": 2484873,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766962.967, "dur": 1.373,
    "args": {
      "External id": 2484874,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766963.449, "dur": 0.752,
    "args": {
      "External id": 2484875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766968.748, "dur": 5.850,
    "args": {
      "External id": 2484876,"Record function id": 0, "Ev Idx": 1675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766969.611, "dur": 4.569,
    "args": {
      "External id": 2484877,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766970.503, "dur": 3.097,
    "args": {
      "External id": 2484878,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766970.985, "dur": 2.525,
    "args": {
      "External id": 2484879,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766977.741, "dur": 3.529,
    "args": {
      "External id": 2484880,"Record function id": 0, "Ev Idx": 1679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766978.718, "dur": 2.112,
    "args": {
      "External id": 2484881,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766979.216, "dur": 1.187,
    "args": {
      "External id": 2484882,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656766979.593, "dur": 0.729,
    "args": {
      "External id": 2484883,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767031.552, "dur": 7.710,
    "args": {
      "External id": 2484884,"Record function id": 0, "Ev Idx": 1683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767033.950, "dur": 4.634,
    "args": {
      "External id": 2484885,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767035.232, "dur": 2.336,
    "args": {
      "External id": 2484886,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767036.067, "dur": 1.286,
    "args": {
      "External id": 2484887,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767043.987, "dur": 4.390,
    "args": {
      "External id": 2484888,"Record function id": 0, "Ev Idx": 1687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767045.364, "dur": 2.572,
    "args": {
      "External id": 2484889,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767046.002, "dur": 1.293,
    "args": {
      "External id": 2484890,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767046.617, "dur": 0.612,
    "args": {
      "External id": 2484891,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767051.826, "dur": 6.104,
    "args": {
      "External id": 2484892,"Record function id": 0, "Ev Idx": 1691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767053.149, "dur": 4.360,
    "args": {
      "External id": 2484893,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767053.650, "dur": 3.245,
    "args": {
      "External id": 2484894,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767056.126, "dur": 0.703,
    "args": {
      "External id": 2484895,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767062.542, "dur": 3.677,
    "args": {
      "External id": 2484896,"Record function id": 0, "Ev Idx": 1695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767063.510, "dur": 2.258,
    "args": {
      "External id": 2484897,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767064.050, "dur": 1.120,
    "args": {
      "External id": 2484898,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767064.366, "dur": 0.738,
    "args": {
      "External id": 2484899,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767070.482, "dur": 161768.749,
    "args": {
      "External id": 2484900,"Record function id": 0, "Sequence number": 24740475, "Fwd thread id": 1, "Ev Idx": 1699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767071.853, "dur": 161757.898,
    "args": {
      "External id": 2484901,"Sequence number": 24740475, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1700
    }
  },
  {
    "ph": "f", "id": 173, "pid": 1336757, "tid": 1381170, "ts": 1514656767071.853,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.27)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767107.837, "dur": 43.498,
    "args": {
      "External id": 2484902,"Record function id": 0, "Ev Idx": 1701
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.27)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767159.081, "dur": 91.540,
    "args": {
      "External id": 2484903,"Record function id": 0, "Ev Idx": 1702
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.27)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767257.632, "dur": 161563.893,
    "args": {
      "External id": 2484904,"Record function id": 0, "Ev Idx": 1703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767321.009, "dur": 7.385,
    "args": {
      "External id": 2484905,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767340.329, "dur": 4.911,
    "args": {
      "External id": 2484906,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767364.668, "dur": 160567.753,
    "args": {
      "External id": 2484907,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767380.473, "dur": 160541.926,
    "args": {
      "External id": 2484908,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767464.853, "dur": 6.137,
    "args": {
      "External id": 2484909,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767483.113, "dur": 160396.408,
    "args": {
      "External id": 2484910,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 1709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767485.725, "dur": 160392.846,
    "args": {
      "External id": 2484911,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 1710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767488.799, "dur": 7.502,
    "args": {
      "External id": 2484912,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656767498.078, "dur": 160376.186,
    "args": {
      "External id": 2484913,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 1712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928069.545, "dur": 10.089,
    "args": {
      "External id": 2484914,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 1713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928072.325, "dur": 6.682,
    "args": {
      "External id": 2484915,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928118.191, "dur": 357.825,
    "args": {
      "External id": 2484916,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 1715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928154.261, "dur": 317.391,
    "args": {
      "External id": 2484917,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1716, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928169.893, "dur": 296.136,
    "args": {
      "External id": 2484918,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 1717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928499.817, "dur": 2.031,
    "args": {
      "External id": 2484919,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1718, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928587.243, "dur": 8.099,
    "args": {
      "External id": 2484920,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928649.821, "dur": 1.417,
    "args": {
      "External id": 2484921,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928667.688, "dur": 1.702,
    "args": {
      "External id": 2484922,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928685.037, "dur": 1.031,
    "args": {
      "External id": 2484923,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928698.350, "dur": 0.928,
    "args": {
      "External id": 2484924,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928708.698, "dur": 0.936,
    "args": {
      "External id": 2484925,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928722.524, "dur": 0.972,
    "args": {
      "External id": 2484926,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928738.594, "dur": 2.609,
    "args": {
      "External id": 2484927,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928752.601, "dur": 1.006,
    "args": {
      "External id": 2484928,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928854.092, "dur": 2913.908,
    "args": {
      "External id": 2484929,"Record function id": 0, "Ev Idx": 1728
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.26)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928875.379, "dur": 1161.904,
    "args": {
      "External id": 2484930,"Record function id": 0, "Ev Idx": 1729
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.26)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928892.330, "dur": 390.185,
    "args": {
      "External id": 2484931,"Record function id": 0, "Ev Idx": 1730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656928980.917, "dur": 38.772,
    "args": {
      "External id": 2484932,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929025.719, "dur": 1.170,
    "args": {
      "External id": 2484933,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929028.542, "dur": 1.019,
    "args": {
      "External id": 2484934,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929031.415, "dur": 3.488,
    "args": {
      "External id": 2484935,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929038.396, "dur": 0.892,
    "args": {
      "External id": 2484936,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929040.881, "dur": 1.200,
    "args": {
      "External id": 2484937,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929043.510, "dur": 2.886,
    "args": {
      "External id": 2484938,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929047.925, "dur": 0.894,
    "args": {
      "External id": 2484939,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929053.901, "dur": 0.944,
    "args": {
      "External id": 2484940,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929056.292, "dur": 0.961,
    "args": {
      "External id": 2484941,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929078.592, "dur": 173.120,
    "args": {
      "External id": 2484942,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929100.036, "dur": 147.088,
    "args": {
      "External id": 2484943,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929125.640, "dur": 12.948,
    "args": {
      "External id": 2484944,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929143.115, "dur": 71.768,
    "args": {
      "External id": 2484945,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 1744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929147.534, "dur": 66.985,
    "args": {
      "External id": 2484946,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 1745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929151.011, "dur": 9.592,
    "args": {
      "External id": 2484947,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929162.206, "dur": 51.788,
    "args": {
      "External id": 2484948,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 1747
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.25", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929378.193, "dur": 649.464,
    "args": {
      "External id": 2484949,"Record function id": 0, "Ev Idx": 1748
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.25)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929395.573, "dur": 579.846,
    "args": {
      "External id": 2484950,"Record function id": 0, "Ev Idx": 1749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929457.602, "dur": 4.665,
    "args": {
      "External id": 2484951,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929478.452, "dur": 40.217,
    "args": {
      "External id": 2484952,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929484.967, "dur": 3.198,
    "args": {
      "External id": 2484953,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929489.588, "dur": 0.565,
    "args": {
      "External id": 2484954,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929491.160, "dur": 0.345,
    "args": {
      "External id": 2484955,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929496.574, "dur": 0.426,
    "args": {
      "External id": 2484956,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929497.766, "dur": 0.651,
    "args": {
      "External id": 2484957,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929499.325, "dur": 0.653,
    "args": {
      "External id": 2484958,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929504.060, "dur": 2.068,
    "args": {
      "External id": 2484959,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929506.854, "dur": 0.388,
    "args": {
      "External id": 2484960,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929509.986, "dur": 2.041,
    "args": {
      "External id": 2484961,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929555.460, "dur": 35.488,
    "args": {
      "External id": 2484962,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929627.000, "dur": 110.499,
    "args": {
      "External id": 2484963,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 1762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929637.981, "dur": 5.544,
    "args": {
      "External id": 2484964,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929648.738, "dur": 12.423,
    "args": {
      "External id": 2484965,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929652.975, "dur": 7.740,
    "args": {
      "External id": 2484966,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 1765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929658.417, "dur": 0.671,
    "args": {
      "External id": 2484967,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929667.743, "dur": 30.065,
    "args": {
      "External id": 2484968,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929669.126, "dur": 0.445,
    "args": {
      "External id": 2484969,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929672.675, "dur": 2.008,
    "args": {
      "External id": 2484970,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929675.597, "dur": 0.231,
    "args": {
      "External id": 2484971,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929676.376, "dur": 2.654,
    "args": {
      "External id": 2484972,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929684.323, "dur": 0.270,
    "args": {
      "External id": 2484973,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929685.541, "dur": 0.324,
    "args": {
      "External id": 2484974,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929688.347, "dur": 0.453,
    "args": {
      "External id": 2484975,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929690.856, "dur": 0.328,
    "args": {
      "External id": 2484976,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929691.909, "dur": 0.376,
    "args": {
      "External id": 2484977,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929710.789, "dur": 18.410,
    "args": {
      "External id": 2484978,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929784.549, "dur": 124.872,
    "args": {
      "External id": 2484979,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 1778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929816.613, "dur": 89.885,
    "args": {
      "External id": 2484980,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1779, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929827.635, "dur": 74.899,
    "args": {
      "External id": 2484981,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 1780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514656929926.757, "dur": 1.815,
    "args": {
      "External id": 2484982,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1781, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656930045.294, "dur": 1698.058,
    "args": {
      "External id": 2484983,"Sequence number": 24740474, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1782
    }
  },
  {
    "ph": "f", "id": 174, "pid": 1336757, "tid": 1381170, "ts": 1514656930045.294,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656930163.053, "dur": 106.579,
    "args": {
      "External id": 2484984,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 1783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656930313.914, "dur": 41.602,
    "args": {
      "External id": 2484985,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 1784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656930372.709, "dur": 48.292,
    "args": {
      "External id": 2484986,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 1785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656930430.951, "dur": 31.984,
    "args": {
      "External id": 2484987,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656930470.975, "dur": 46.424,
    "args": {
      "External id": 2484988,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656930524.135, "dur": 48.480,
    "args": {
      "External id": 2484989,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656930584.454, "dur": 43.187,
    "args": {
      "External id": 2484990,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514656930654.554, "dur": 26.187,
    "args": {
      "External id": 2484991,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 1790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656930706.399, "dur": 31.452,
    "args": {
      "External id": 2484992,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514656930759.921, "dur": 20.301,
    "args": {
      "External id": 2484993,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514656930798.925, "dur": 17.752,
    "args": {
      "External id": 2484994,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656930824.644, "dur": 30.587,
    "args": {
      "External id": 2484995,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656930858.293, "dur": 38.823,
    "args": {
      "External id": 2484996,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656930927.926, "dur": 216.856,
    "args": {
      "External id": 2484997,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931048.296, "dur": 6.593,
    "args": {
      "External id": 2484998,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931057.032, "dur": 2.233,
    "args": {
      "External id": 2484999,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931187.131, "dur": 26.218,
    "args": {
      "External id": 2485000,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931227.220, "dur": 16.205,
    "args": {
      "External id": 2485001,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931251.266, "dur": 59.068,
    "args": {
      "External id": 2485002,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931321.406, "dur": 38.795,
    "args": {
      "External id": 2485003,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931372.795, "dur": 20.960,
    "args": {
      "External id": 2485004,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931398.081, "dur": 34.920,
    "args": {
      "External id": 2485005,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931438.117, "dur": 20.135,
    "args": {
      "External id": 2485006,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931464.883, "dur": 32.378,
    "args": {
      "External id": 2485007,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931520.398, "dur": 39.199,
    "args": {
      "External id": 2485008,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 1807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931581.582, "dur": 28.686,
    "args": {
      "External id": 2485009,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931627.724, "dur": 18.402,
    "args": {
      "External id": 2485010,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931666.985, "dur": 15.293,
    "args": {
      "External id": 2485011,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931696.684, "dur": 18.415,
    "args": {
      "External id": 2485012,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 1811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931790.811, "dur": 15.501,
    "args": {
      "External id": 2485013,"Record function id": 0, "Ev Idx": 1812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931794.004, "dur": 11.424,
    "args": {
      "External id": 2485014,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931798.421, "dur": 5.998,
    "args": {
      "External id": 2485015,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931799.946, "dur": 4.330,
    "args": {
      "External id": 2485016,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931810.379, "dur": 4.986,
    "args": {
      "External id": 2485017,"Record function id": 0, "Ev Idx": 1816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931811.650, "dur": 3.248,
    "args": {
      "External id": 2485018,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931813.034, "dur": 1.307,
    "args": {
      "External id": 2485019,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931813.505, "dur": 0.748,
    "args": {
      "External id": 2485020,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931818.660, "dur": 6.523,
    "args": {
      "External id": 2485021,"Record function id": 0, "Ev Idx": 1820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931819.997, "dur": 4.752,
    "args": {
      "External id": 2485022,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931820.693, "dur": 3.595,
    "args": {
      "External id": 2485023,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931821.265, "dur": 2.917,
    "args": {
      "External id": 2485024,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931828.443, "dur": 3.828,
    "args": {
      "External id": 2485025,"Record function id": 0, "Ev Idx": 1824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931829.558, "dur": 2.311,
    "args": {
      "External id": 2485026,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931830.282, "dur": 1.148,
    "args": {
      "External id": 2485027,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931830.574, "dur": 0.793,
    "args": {
      "External id": 2485028,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931835.404, "dur": 3.925,
    "args": {
      "External id": 2485029,"Record function id": 0, "Ev Idx": 1828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931836.659, "dur": 2.268,
    "args": {
      "External id": 2485030,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931837.234, "dur": 1.207,
    "args": {
      "External id": 2485031,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931837.717, "dur": 0.660,
    "args": {
      "External id": 2485032,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931842.410, "dur": 5.653,
    "args": {
      "External id": 2485033,"Record function id": 0, "Ev Idx": 1832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931845.122, "dur": 2.481,
    "args": {
      "External id": 2485034,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931845.967, "dur": 1.060,
    "args": {
      "External id": 2485035,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931846.340, "dur": 0.623,
    "args": {
      "External id": 2485036,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931851.287, "dur": 3.985,
    "args": {
      "External id": 2485037,"Record function id": 0, "Ev Idx": 1836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931852.216, "dur": 2.660,
    "args": {
      "External id": 2485038,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931853.181, "dur": 1.165,
    "args": {
      "External id": 2485039,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931853.465, "dur": 0.816,
    "args": {
      "External id": 2485040,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931858.361, "dur": 6.605,
    "args": {
      "External id": 2485041,"Record function id": 0, "Ev Idx": 1840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931859.549, "dur": 4.995,
    "args": {
      "External id": 2485042,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931860.641, "dur": 3.237,
    "args": {
      "External id": 2485043,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931863.081, "dur": 0.731,
    "args": {
      "External id": 2485044,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931868.026, "dur": 4.667,
    "args": {
      "External id": 2485045,"Record function id": 0, "Ev Idx": 1844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931869.132, "dur": 3.186,
    "args": {
      "External id": 2485046,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931870.269, "dur": 1.406,
    "args": {
      "External id": 2485047,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931870.938, "dur": 0.675,
    "args": {
      "External id": 2485048,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931876.665, "dur": 42224.755,
    "args": {
      "External id": 2485049,"Record function id": 0, "Sequence number": 24740473, "Fwd thread id": 1, "Ev Idx": 1848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931877.829, "dur": 42215.082,
    "args": {
      "External id": 2485050,"Sequence number": 24740473, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1849
    }
  },
  {
    "ph": "f", "id": 175, "pid": 1336757, "tid": 1381170, "ts": 1514656931877.829,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.26)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931907.277, "dur": 36.945,
    "args": {
      "External id": 2485051,"Record function id": 0, "Ev Idx": 1850
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.26)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656931951.357, "dur": 104.423,
    "args": {
      "External id": 2485052,"Record function id": 0, "Ev Idx": 1851
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.26)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656932066.099, "dur": 42018.630,
    "args": {
      "External id": 2485053,"Record function id": 0, "Ev Idx": 1852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656932157.902, "dur": 7.481,
    "args": {
      "External id": 2485054,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656932174.951, "dur": 6.359,
    "args": {
      "External id": 2485055,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514656932198.192, "dur": 41010.384,
    "args": {
      "External id": 2485056,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514656932211.101, "dur": 40988.652,
    "args": {
      "External id": 2485057,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 1856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656932268.463, "dur": 14.352,
    "args": {
      "External id": 2485058,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656932291.180, "dur": 40871.861,
    "args": {
      "External id": 2485059,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 1858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656932293.522, "dur": 40868.886,
    "args": {
      "External id": 2485060,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 1859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656932298.704, "dur": 5.411,
    "args": {
      "External id": 2485061,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656932305.990, "dur": 40852.614,
    "args": {
      "External id": 2485062,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 1861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656973296.549, "dur": 9.088,
    "args": {
      "External id": 2485063,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 1862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656973299.273, "dur": 5.953,
    "args": {
      "External id": 2485064,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656973332.791, "dur": 413.802,
    "args": {
      "External id": 2485065,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 1864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514656973365.649, "dur": 375.776,
    "args": {
      "External id": 2485066,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1865, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514656973379.922, "dur": 354.639,
    "args": {
      "External id": 2485067,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 1866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514656973772.139, "dur": 2.654,
    "args": {
      "External id": 2485068,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1867, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656973836.925, "dur": 6.380,
    "args": {
      "External id": 2485069,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656973889.100, "dur": 1.426,
    "args": {
      "External id": 2485070,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656973905.692, "dur": 3.334,
    "args": {
      "External id": 2485071,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656973922.379, "dur": 0.830,
    "args": {
      "External id": 2485072,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656973935.558, "dur": 0.693,
    "args": {
      "External id": 2485073,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656973947.856, "dur": 1.090,
    "args": {
      "External id": 2485074,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656973959.038, "dur": 2.806,
    "args": {
      "External id": 2485075,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656973971.970, "dur": 2.639,
    "args": {
      "External id": 2485076,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974018.494, "dur": 1.605,
    "args": {
      "External id": 2485077,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974116.315, "dur": 2794.838,
    "args": {
      "External id": 2485078,"Record function id": 0, "Ev Idx": 1877
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.25)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974135.300, "dur": 1070.782,
    "args": {
      "External id": 2485079,"Record function id": 0, "Ev Idx": 1878
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.25)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974149.944, "dur": 322.536,
    "args": {
      "External id": 2485080,"Record function id": 0, "Ev Idx": 1879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974234.127, "dur": 4.120,
    "args": {
      "External id": 2485081,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 1880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974241.187, "dur": 1.041,
    "args": {
      "External id": 2485082,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974244.092, "dur": 2.767,
    "args": {
      "External id": 2485083,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974248.524, "dur": 0.943,
    "args": {
      "External id": 2485084,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974252.796, "dur": 1.044,
    "args": {
      "External id": 2485085,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974255.060, "dur": 0.991,
    "args": {
      "External id": 2485086,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 1885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974258.942, "dur": 2.866,
    "args": {
      "External id": 2485087,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 1886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974263.264, "dur": 0.916,
    "args": {
      "External id": 2485088,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974267.514, "dur": 1.114,
    "args": {
      "External id": 2485089,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974269.891, "dur": 1.097,
    "args": {
      "External id": 2485090,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 1889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974287.947, "dur": 156.399,
    "args": {
      "External id": 2485091,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974305.089, "dur": 134.648,
    "args": {
      "External id": 2485092,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 1891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974324.213, "dur": 16.640,
    "args": {
      "External id": 2485093,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974344.791, "dur": 66.526,
    "args": {
      "External id": 2485094,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 1893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974349.690, "dur": 61.273,
    "args": {
      "External id": 2485095,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 1894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974352.742, "dur": 5.389,
    "args": {
      "External id": 2485096,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974360.033, "dur": 50.487,
    "args": {
      "External id": 2485097,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 1896
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.24", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974578.347, "dur": 620.296,
    "args": {
      "External id": 2485098,"Record function id": 0, "Ev Idx": 1897
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.24)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974596.626, "dur": 589.342,
    "args": {
      "External id": 2485099,"Record function id": 0, "Ev Idx": 1898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974660.945, "dur": 5.541,
    "args": {
      "External id": 2485100,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974682.287, "dur": 36.339,
    "args": {
      "External id": 2485101,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974688.118, "dur": 1.819,
    "args": {
      "External id": 2485102,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974692.983, "dur": 0.448,
    "args": {
      "External id": 2485103,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974694.003, "dur": 0.346,
    "args": {
      "External id": 2485104,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974697.068, "dur": 1.726,
    "args": {
      "External id": 2485105,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974699.615, "dur": 0.429,
    "args": {
      "External id": 2485106,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974702.289, "dur": 2.351,
    "args": {
      "External id": 2485107,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974706.928, "dur": 0.364,
    "args": {
      "External id": 2485108,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974708.046, "dur": 0.283,
    "args": {
      "External id": 2485109,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974711.101, "dur": 0.382,
    "args": {
      "External id": 2485110,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974728.940, "dur": 33.634,
    "args": {
      "External id": 2485111,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974792.548, "dur": 114.214,
    "args": {
      "External id": 2485112,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 1911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974802.322, "dur": 3.636,
    "args": {
      "External id": 2485113,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974810.548, "dur": 12.826,
    "args": {
      "External id": 2485114,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974814.546, "dur": 8.356,
    "args": {
      "External id": 2485115,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 1914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974819.187, "dur": 2.393,
    "args": {
      "External id": 2485116,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 1915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974829.672, "dur": 38.104,
    "args": {
      "External id": 2485117,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 1916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974830.815, "dur": 0.474,
    "args": {
      "External id": 2485118,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974834.240, "dur": 0.636,
    "args": {
      "External id": 2485119,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974845.897, "dur": 2.334,
    "args": {
      "External id": 2485120,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974851.750, "dur": 0.603,
    "args": {
      "External id": 2485121,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974853.010, "dur": 0.250,
    "args": {
      "External id": 2485122,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974853.741, "dur": 1.649,
    "args": {
      "External id": 2485123,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974857.144, "dur": 0.335,
    "args": {
      "External id": 2485124,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974857.994, "dur": 0.408,
    "args": {
      "External id": 2485125,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974860.672, "dur": 0.342,
    "args": {
      "External id": 2485126,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 1925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974879.142, "dur": 20.224,
    "args": {
      "External id": 2485127,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 1926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974950.662, "dur": 161.092,
    "args": {
      "External id": 2485128,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 1927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514656974980.813, "dur": 126.935,
    "args": {
      "External id": 2485129,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 1928, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514656975025.698, "dur": 77.318,
    "args": {
      "External id": 2485130,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 1929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514656975129.586, "dur": 1.813,
    "args": {
      "External id": 2485131,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 1930, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656975213.789, "dur": 1673.757,
    "args": {
      "External id": 2485132,"Sequence number": 24740472, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1931
    }
  },
  {
    "ph": "f", "id": 176, "pid": 1336757, "tid": 1381170, "ts": 1514656975213.789,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656975332.331, "dur": 101.896,
    "args": {
      "External id": 2485133,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 1932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656975476.969, "dur": 40.178,
    "args": {
      "External id": 2485134,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 1933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656975551.592, "dur": 57.098,
    "args": {
      "External id": 2485135,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 1934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656975621.534, "dur": 32.822,
    "args": {
      "External id": 2485136,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656975661.028, "dur": 45.791,
    "args": {
      "External id": 2485137,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656975714.251, "dur": 28.092,
    "args": {
      "External id": 2485138,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 1937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656975756.138, "dur": 42.836,
    "args": {
      "External id": 2485139,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 1938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514656975830.415, "dur": 26.586,
    "args": {
      "External id": 2485140,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 1939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514656975875.921, "dur": 30.520,
    "args": {
      "External id": 2485141,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514656975927.680, "dur": 19.618,
    "args": {
      "External id": 2485142,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514656975965.238, "dur": 51.351,
    "args": {
      "External id": 2485143,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976029.781, "dur": 35.379,
    "args": {
      "External id": 2485144,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976068.876, "dur": 33.502,
    "args": {
      "External id": 2485145,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976131.545, "dur": 174.397,
    "args": {
      "External id": 2485146,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 1945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976212.077, "dur": 8.102,
    "args": {
      "External id": 2485147,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976222.076, "dur": 2.094,
    "args": {
      "External id": 2485148,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 1947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976343.173, "dur": 24.687,
    "args": {
      "External id": 2485149,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976381.227, "dur": 16.241,
    "args": {
      "External id": 2485150,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976404.911, "dur": 32.636,
    "args": {
      "External id": 2485151,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976442.824, "dur": 34.450,
    "args": {
      "External id": 2485152,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976486.395, "dur": 22.895,
    "args": {
      "External id": 2485153,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976513.522, "dur": 61.604,
    "args": {
      "External id": 2485154,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976593.524, "dur": 29.845,
    "args": {
      "External id": 2485155,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 1954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976630.349, "dur": 32.355,
    "args": {
      "External id": 2485156,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 1955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976690.640, "dur": 25.244,
    "args": {
      "External id": 2485157,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 1956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976736.563, "dur": 26.410,
    "args": {
      "External id": 2485158,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 1957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976778.541, "dur": 18.715,
    "args": {
      "External id": 2485159,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 1958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976813.007, "dur": 15.704,
    "args": {
      "External id": 2485160,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 1959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976843.387, "dur": 17.989,
    "args": {
      "External id": 2485161,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 1960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976933.094, "dur": 14.411,
    "args": {
      "External id": 2485162,"Record function id": 0, "Ev Idx": 1961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976936.124, "dur": 10.510,
    "args": {
      "External id": 2485163,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976940.157, "dur": 5.685,
    "args": {
      "External id": 2485164,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976941.806, "dur": 3.928,
    "args": {
      "External id": 2485165,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976951.323, "dur": 8.040,
    "args": {
      "External id": 2485166,"Record function id": 0, "Ev Idx": 1965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976954.025, "dur": 4.862,
    "args": {
      "External id": 2485167,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976954.785, "dur": 3.235,
    "args": {
      "External id": 2485168,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976955.265, "dur": 2.665,
    "args": {
      "External id": 2485169,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976962.829, "dur": 4.975,
    "args": {
      "External id": 2485170,"Record function id": 0, "Ev Idx": 1969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976964.021, "dur": 3.322,
    "args": {
      "External id": 2485171,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976964.509, "dur": 2.285,
    "args": {
      "External id": 2485172,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976965.308, "dur": 1.259,
    "args": {
      "External id": 2485173,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 1972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976970.908, "dur": 4.102,
    "args": {
      "External id": 2485174,"Record function id": 0, "Ev Idx": 1973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976972.050, "dur": 2.559,
    "args": {
      "External id": 2485175,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976972.839, "dur": 1.382,
    "args": {
      "External id": 2485176,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976973.584, "dur": 0.571,
    "args": {
      "External id": 2485177,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 1976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976978.094, "dur": 38.357,
    "args": {
      "External id": 2485178,"Record function id": 0, "Ev Idx": 1977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976979.042, "dur": 36.207,
    "args": {
      "External id": 2485179,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976979.729, "dur": 33.967,
    "args": {
      "External id": 2485180,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656976980.011, "dur": 0.618,
    "args": {
      "External id": 2485181,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977022.165, "dur": 5.711,
    "args": {
      "External id": 2485182,"Record function id": 0, "Ev Idx": 1981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977023.616, "dur": 3.798,
    "args": {
      "External id": 2485183,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977024.886, "dur": 2.043,
    "args": {
      "External id": 2485184,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977025.652, "dur": 1.211,
    "args": {
      "External id": 2485185,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977031.540, "dur": 5.273,
    "args": {
      "External id": 2485186,"Record function id": 0, "Ev Idx": 1985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977032.508, "dur": 3.887,
    "args": {
      "External id": 2485187,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977033.061, "dur": 2.921,
    "args": {
      "External id": 2485188,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977035.320, "dur": 0.599,
    "args": {
      "External id": 2485189,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977040.026, "dur": 4.086,
    "args": {
      "External id": 2485190,"Record function id": 0, "Ev Idx": 1989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977041.069, "dur": 2.621,
    "args": {
      "External id": 2485191,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977041.718, "dur": 1.408,
    "args": {
      "External id": 2485192,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977042.272, "dur": 0.793,
    "args": {
      "External id": 2485193,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 1992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977047.132, "dur": 5.489,
    "args": {
      "External id": 2485194,"Record function id": 0, "Ev Idx": 1993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977048.160, "dur": 4.053,
    "args": {
      "External id": 2485195,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977048.617, "dur": 3.165,
    "args": {
      "External id": 2485196,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977048.903, "dur": 2.815,
    "args": {
      "External id": 2485197,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 1996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977056.203, "dur": 40440.557,
    "args": {
      "External id": 2485198,"Record function id": 0, "Sequence number": 24740471, "Fwd thread id": 1, "Ev Idx": 1997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977057.724, "dur": 40429.321,
    "args": {
      "External id": 2485199,"Sequence number": 24740471, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 1998
    }
  },
  {
    "ph": "f", "id": 177, "pid": 1336757, "tid": 1381170, "ts": 1514656977057.724,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.25)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977087.775, "dur": 35.121,
    "args": {
      "External id": 2485200,"Record function id": 0, "Ev Idx": 1999
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.25)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977130.588, "dur": 64.777,
    "args": {
      "External id": 2485201,"Record function id": 0, "Ev Idx": 2000
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.25)", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977205.125, "dur": 40273.117,
    "args": {
      "External id": 2485202,"Record function id": 0, "Ev Idx": 2001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977293.208, "dur": 7.498,
    "args": {
      "External id": 2485203,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977310.819, "dur": 4.959,
    "args": {
      "External id": 2485204,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977328.981, "dur": 39333.778,
    "args": {
      "External id": 2485205,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977342.474, "dur": 39311.659,
    "args": {
      "External id": 2485206,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977392.440, "dur": 13.800,
    "args": {
      "External id": 2485207,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977413.048, "dur": 39203.147,
    "args": {
      "External id": 2485208,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977415.731, "dur": 39199.717,
    "args": {
      "External id": 2485209,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977418.929, "dur": 6.554,
    "args": {
      "External id": 2485210,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514656977427.011, "dur": 39184.720,
    "args": {
      "External id": 2485211,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657016748.644, "dur": 8.382,
    "args": {
      "External id": 2485212,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657016751.404, "dur": 5.271,
    "args": {
      "External id": 2485213,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657016785.342, "dur": 393.152,
    "args": {
      "External id": 2485214,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657016819.247, "dur": 354.102,
    "args": {
      "External id": 2485215,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2014, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657016833.794, "dur": 332.813,
    "args": {
      "External id": 2485216,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017203.792, "dur": 2.278,
    "args": {
      "External id": 2485217,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2016, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017261.945, "dur": 7.060,
    "args": {
      "External id": 2485218,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017314.440, "dur": 2.831,
    "args": {
      "External id": 2485219,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017335.263, "dur": 1.424,
    "args": {
      "External id": 2485220,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017349.818, "dur": 0.990,
    "args": {
      "External id": 2485221,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017364.455, "dur": 0.934,
    "args": {
      "External id": 2485222,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017377.053, "dur": 2.545,
    "args": {
      "External id": 2485223,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017391.118, "dur": 1.007,
    "args": {
      "External id": 2485224,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017404.615, "dur": 3.272,
    "args": {
      "External id": 2485225,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017419.910, "dur": 0.928,
    "args": {
      "External id": 2485226,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017510.480, "dur": 2812.215,
    "args": {
      "External id": 2485227,"Record function id": 0, "Ev Idx": 2026
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.24)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017543.831, "dur": 1075.643,
    "args": {
      "External id": 2485228,"Record function id": 0, "Ev Idx": 2027
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.24)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017560.975, "dur": 323.758,
    "args": {
      "External id": 2485229,"Record function id": 0, "Ev Idx": 2028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017646.445, "dur": 6.325,
    "args": {
      "External id": 2485230,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017656.067, "dur": 1.476,
    "args": {
      "External id": 2485231,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017659.423, "dur": 1.117,
    "args": {
      "External id": 2485232,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017662.540, "dur": 0.975,
    "args": {
      "External id": 2485233,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017666.618, "dur": 1.167,
    "args": {
      "External id": 2485234,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017669.636, "dur": 0.793,
    "args": {
      "External id": 2485235,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017672.112, "dur": 2.892,
    "args": {
      "External id": 2485236,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017676.389, "dur": 1.151,
    "args": {
      "External id": 2485237,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017680.891, "dur": 2.607,
    "args": {
      "External id": 2485238,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017685.453, "dur": 0.914,
    "args": {
      "External id": 2485239,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017703.092, "dur": 150.899,
    "args": {
      "External id": 2485240,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017718.356, "dur": 131.398,
    "args": {
      "External id": 2485241,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017739.347, "dur": 11.474,
    "args": {
      "External id": 2485242,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017754.450, "dur": 66.722,
    "args": {
      "External id": 2485243,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017758.734, "dur": 62.170,
    "args": {
      "External id": 2485244,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017762.853, "dur": 6.047,
    "args": {
      "External id": 2485245,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017770.636, "dur": 49.787,
    "args": {
      "External id": 2485246,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2045
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.23", "pid": 1336757, "tid": 1381170,
    "ts": 1514657017969.540, "dur": 641.726,
    "args": {
      "External id": 2485247,"Record function id": 0, "Ev Idx": 2046
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.23)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018021.102, "dur": 577.978,
    "args": {
      "External id": 2485248,"Record function id": 0, "Ev Idx": 2047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018108.495, "dur": 6.849,
    "args": {
      "External id": 2485249,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018131.957, "dur": 36.662,
    "args": {
      "External id": 2485250,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018136.481, "dur": 3.718,
    "args": {
      "External id": 2485251,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018142.423, "dur": 0.332,
    "args": {
      "External id": 2485252,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018144.234, "dur": 0.408,
    "args": {
      "External id": 2485253,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018147.555, "dur": 2.359,
    "args": {
      "External id": 2485254,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018151.262, "dur": 0.366,
    "args": {
      "External id": 2485255,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018153.048, "dur": 0.415,
    "args": {
      "External id": 2485256,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018156.655, "dur": 0.271,
    "args": {
      "External id": 2485257,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018158.305, "dur": 0.298,
    "args": {
      "External id": 2485258,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018160.084, "dur": 1.970,
    "args": {
      "External id": 2485259,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018179.873, "dur": 36.573,
    "args": {
      "External id": 2485260,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018248.780, "dur": 103.652,
    "args": {
      "External id": 2485261,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018258.850, "dur": 3.238,
    "args": {
      "External id": 2485262,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018267.085, "dur": 9.357,
    "args": {
      "External id": 2485263,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018270.948, "dur": 5.095,
    "args": {
      "External id": 2485264,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018274.262, "dur": 0.514,
    "args": {
      "External id": 2485265,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018283.061, "dur": 31.172,
    "args": {
      "External id": 2485266,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018284.982, "dur": 2.106,
    "args": {
      "External id": 2485267,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018288.901, "dur": 0.324,
    "args": {
      "External id": 2485268,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018291.034, "dur": 1.783,
    "args": {
      "External id": 2485269,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018293.985, "dur": 0.321,
    "args": {
      "External id": 2485270,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018295.807, "dur": 0.333,
    "args": {
      "External id": 2485271,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018298.609, "dur": 0.367,
    "args": {
      "External id": 2485272,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018300.442, "dur": 0.312,
    "args": {
      "External id": 2485273,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018302.460, "dur": 0.291,
    "args": {
      "External id": 2485274,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018305.808, "dur": 2.289,
    "args": {
      "External id": 2485275,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018325.677, "dur": 18.660,
    "args": {
      "External id": 2485276,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018395.569, "dur": 116.099,
    "args": {
      "External id": 2485277,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018426.193, "dur": 82.608,
    "args": {
      "External id": 2485278,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2077, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018435.500, "dur": 69.319,
    "args": {
      "External id": 2485279,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018544.019, "dur": 2.891,
    "args": {
      "External id": 2485280,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2079, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018628.474, "dur": 1673.481,
    "args": {
      "External id": 2485281,"Sequence number": 24740470, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2080
    }
  },
  {
    "ph": "f", "id": 178, "pid": 1336757, "tid": 1381170, "ts": 1514657018628.474,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018737.346, "dur": 103.352,
    "args": {
      "External id": 2485282,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018881.226, "dur": 41.024,
    "args": {
      "External id": 2485283,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657018940.298, "dur": 88.334,
    "args": {
      "External id": 2485284,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019043.289, "dur": 37.925,
    "args": {
      "External id": 2485285,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019087.675, "dur": 46.545,
    "args": {
      "External id": 2485286,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019141.708, "dur": 27.605,
    "args": {
      "External id": 2485287,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019180.418, "dur": 42.142,
    "args": {
      "External id": 2485288,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019250.255, "dur": 25.975,
    "args": {
      "External id": 2485289,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019297.306, "dur": 31.589,
    "args": {
      "External id": 2485290,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019349.210, "dur": 21.300,
    "args": {
      "External id": 2485291,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019386.256, "dur": 15.112,
    "args": {
      "External id": 2485292,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019411.091, "dur": 29.049,
    "args": {
      "External id": 2485293,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019443.361, "dur": 32.240,
    "args": {
      "External id": 2485294,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019510.532, "dur": 195.038,
    "args": {
      "External id": 2485295,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019610.349, "dur": 6.668,
    "args": {
      "External id": 2485296,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019619.164, "dur": 2.168,
    "args": {
      "External id": 2485297,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019741.101, "dur": 24.257,
    "args": {
      "External id": 2485298,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019777.063, "dur": 15.099,
    "args": {
      "External id": 2485299,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019800.231, "dur": 39.355,
    "args": {
      "External id": 2485300,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019846.573, "dur": 35.677,
    "args": {
      "External id": 2485301,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019889.358, "dur": 20.969,
    "args": {
      "External id": 2485302,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019914.099, "dur": 29.448,
    "args": {
      "External id": 2485303,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657019962.355, "dur": 62.392,
    "args": {
      "External id": 2485304,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020040.076, "dur": 36.155,
    "args": {
      "External id": 2485305,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020102.425, "dur": 24.173,
    "args": {
      "External id": 2485306,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020151.477, "dur": 25.036,
    "args": {
      "External id": 2485307,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020192.157, "dur": 17.588,
    "args": {
      "External id": 2485308,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020229.270, "dur": 15.138,
    "args": {
      "External id": 2485309,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020256.162, "dur": 18.343,
    "args": {
      "External id": 2485310,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020344.765, "dur": 16.117,
    "args": {
      "External id": 2485311,"Record function id": 0, "Ev Idx": 2110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020348.347, "dur": 11.720,
    "args": {
      "External id": 2485312,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020352.326, "dur": 6.727,
    "args": {
      "External id": 2485313,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020353.554, "dur": 5.406,
    "args": {
      "External id": 2485314,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020364.702, "dur": 4.174,
    "args": {
      "External id": 2485315,"Record function id": 0, "Ev Idx": 2114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020366.248, "dur": 2.144,
    "args": {
      "External id": 2485316,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020366.820, "dur": 1.140,
    "args": {
      "External id": 2485317,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020367.193, "dur": 0.644,
    "args": {
      "External id": 2485318,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020372.244, "dur": 4.653,
    "args": {
      "External id": 2485319,"Record function id": 0, "Ev Idx": 2118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020373.710, "dur": 2.733,
    "args": {
      "External id": 2485320,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020374.371, "dur": 1.581,
    "args": {
      "External id": 2485321,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020374.905, "dur": 0.970,
    "args": {
      "External id": 2485322,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020380.185, "dur": 3.841,
    "args": {
      "External id": 2485323,"Record function id": 0, "Ev Idx": 2122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020381.384, "dur": 2.219,
    "args": {
      "External id": 2485324,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020381.943, "dur": 1.193,
    "args": {
      "External id": 2485325,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020382.477, "dur": 0.561,
    "args": {
      "External id": 2485326,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020387.200, "dur": 3.638,
    "args": {
      "External id": 2485327,"Record function id": 0, "Ev Idx": 2126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020388.227, "dur": 2.209,
    "args": {
      "External id": 2485328,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020388.997, "dur": 1.037,
    "args": {
      "External id": 2485329,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020389.323, "dur": 0.642,
    "args": {
      "External id": 2485330,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020393.909, "dur": 3.514,
    "args": {
      "External id": 2485331,"Record function id": 0, "Ev Idx": 2130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020394.954, "dur": 2.080,
    "args": {
      "External id": 2485332,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020395.438, "dur": 1.067,
    "args": {
      "External id": 2485333,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020395.789, "dur": 0.612,
    "args": {
      "External id": 2485334,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020400.729, "dur": 6.109,
    "args": {
      "External id": 2485335,"Record function id": 0, "Ev Idx": 2134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020401.923, "dur": 4.421,
    "args": {
      "External id": 2485336,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020402.419, "dur": 3.447,
    "args": {
      "External id": 2485337,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020405.212, "dur": 0.587,
    "args": {
      "External id": 2485338,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020410.011, "dur": 5.380,
    "args": {
      "External id": 2485339,"Record function id": 0, "Ev Idx": 2138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020411.115, "dur": 3.878,
    "args": {
      "External id": 2485340,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020411.571, "dur": 2.961,
    "args": {
      "External id": 2485341,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020411.870, "dur": 2.583,
    "args": {
      "External id": 2485342,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020418.491, "dur": 4.225,
    "args": {
      "External id": 2485343,"Record function id": 0, "Ev Idx": 2142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020419.906, "dur": 2.379,
    "args": {
      "External id": 2485344,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020420.417, "dur": 1.333,
    "args": {
      "External id": 2485345,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020420.878, "dur": 0.811,
    "args": {
      "External id": 2485346,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020426.686, "dur": 40587.992,
    "args": {
      "External id": 2485347,"Record function id": 0, "Sequence number": 24740469, "Fwd thread id": 1, "Ev Idx": 2146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020428.010, "dur": 40546.025,
    "args": {
      "External id": 2485348,"Sequence number": 24740469, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2147
    }
  },
  {
    "ph": "f", "id": 179, "pid": 1336757, "tid": 1381170, "ts": 1514657020428.010,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.24)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020456.048, "dur": 37.630,
    "args": {
      "External id": 2485349,"Record function id": 0, "Ev Idx": 2148
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.24)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020501.238, "dur": 83.182,
    "args": {
      "External id": 2485350,"Record function id": 0, "Ev Idx": 2149
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.24)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020594.024, "dur": 40372.528,
    "args": {
      "External id": 2485351,"Record function id": 0, "Ev Idx": 2150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020682.540, "dur": 6.554,
    "args": {
      "External id": 2485352,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020698.983, "dur": 4.894,
    "args": {
      "External id": 2485353,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020717.432, "dur": 39400.238,
    "args": {
      "External id": 2485354,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020731.041, "dur": 39377.653,
    "args": {
      "External id": 2485355,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020780.558, "dur": 13.505,
    "args": {
      "External id": 2485356,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020800.318, "dur": 39269.875,
    "args": {
      "External id": 2485357,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020803.759, "dur": 39265.616,
    "args": {
      "External id": 2485358,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020807.678, "dur": 4.720,
    "args": {
      "External id": 2485359,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657020813.918, "dur": 39251.634,
    "args": {
      "External id": 2485360,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657060211.031, "dur": 9.078,
    "args": {
      "External id": 2485361,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657060213.989, "dur": 5.718,
    "args": {
      "External id": 2485362,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657060248.481, "dur": 404.633,
    "args": {
      "External id": 2485363,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657060279.981, "dur": 367.986,
    "args": {
      "External id": 2485364,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2163, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657060291.248, "dur": 350.806,
    "args": {
      "External id": 2485365,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657060679.498, "dur": 2.564,
    "args": {
      "External id": 2485366,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2165, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657060743.584, "dur": 6.900,
    "args": {
      "External id": 2485367,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657060800.705, "dur": 1.551,
    "args": {
      "External id": 2485368,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657060819.661, "dur": 1.057,
    "args": {
      "External id": 2485369,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657060833.831, "dur": 0.935,
    "args": {
      "External id": 2485370,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657060848.713, "dur": 1.109,
    "args": {
      "External id": 2485371,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657060863.768, "dur": 1.052,
    "args": {
      "External id": 2485372,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657060877.329, "dur": 1.067,
    "args": {
      "External id": 2485373,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657060890.183, "dur": 2.368,
    "args": {
      "External id": 2485374,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657060905.299, "dur": 0.807,
    "args": {
      "External id": 2485375,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061032.121, "dur": 2768.174,
    "args": {
      "External id": 2485376,"Record function id": 0, "Ev Idx": 2175
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.23)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061054.051, "dur": 1055.426,
    "args": {
      "External id": 2485377,"Record function id": 0, "Ev Idx": 2176
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.23)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061070.182, "dur": 327.470,
    "args": {
      "External id": 2485378,"Record function id": 0, "Ev Idx": 2177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061157.782, "dur": 4.752,
    "args": {
      "External id": 2485379,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061165.520, "dur": 1.145,
    "args": {
      "External id": 2485380,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061168.453, "dur": 0.954,
    "args": {
      "External id": 2485381,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061171.279, "dur": 1.070,
    "args": {
      "External id": 2485382,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061175.793, "dur": 1.018,
    "args": {
      "External id": 2485383,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061178.363, "dur": 1.114,
    "args": {
      "External id": 2485384,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061181.230, "dur": 2.154,
    "args": {
      "External id": 2485385,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061184.967, "dur": 3.110,
    "args": {
      "External id": 2485386,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061191.419, "dur": 0.741,
    "args": {
      "External id": 2485387,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061193.821, "dur": 0.849,
    "args": {
      "External id": 2485388,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061212.276, "dur": 151.965,
    "args": {
      "External id": 2485389,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061228.249, "dur": 131.335,
    "args": {
      "External id": 2485390,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061247.958, "dur": 14.008,
    "args": {
      "External id": 2485391,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061265.583, "dur": 66.615,
    "args": {
      "External id": 2485392,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061267.883, "dur": 64.060,
    "args": {
      "External id": 2485393,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061271.899, "dur": 5.169,
    "args": {
      "External id": 2485394,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061280.781, "dur": 50.589,
    "args": {
      "External id": 2485395,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2194
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.22", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061487.151, "dur": 614.341,
    "args": {
      "External id": 2485396,"Record function id": 0, "Ev Idx": 2195
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.22)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061503.497, "dur": 586.122,
    "args": {
      "External id": 2485397,"Record function id": 0, "Ev Idx": 2196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061584.559, "dur": 6.471,
    "args": {
      "External id": 2485398,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061607.359, "dur": 36.310,
    "args": {
      "External id": 2485399,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061612.352, "dur": 1.835,
    "args": {
      "External id": 2485400,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061618.145, "dur": 0.384,
    "args": {
      "External id": 2485401,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061620.124, "dur": 2.678,
    "args": {
      "External id": 2485402,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061624.592, "dur": 2.289,
    "args": {
      "External id": 2485403,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061628.285, "dur": 0.476,
    "args": {
      "External id": 2485404,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061630.243, "dur": 0.357,
    "args": {
      "External id": 2485405,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061634.081, "dur": 0.439,
    "args": {
      "External id": 2485406,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061635.788, "dur": 0.588,
    "args": {
      "External id": 2485407,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061637.597, "dur": 0.390,
    "args": {
      "External id": 2485408,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061654.117, "dur": 33.241,
    "args": {
      "External id": 2485409,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061718.738, "dur": 102.120,
    "args": {
      "External id": 2485410,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061728.023, "dur": 3.381,
    "args": {
      "External id": 2485411,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061736.211, "dur": 13.878,
    "args": {
      "External id": 2485412,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061740.297, "dur": 9.327,
    "args": {
      "External id": 2485413,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061744.505, "dur": 4.002,
    "args": {
      "External id": 2485414,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061756.620, "dur": 28.878,
    "args": {
      "External id": 2485415,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061758.940, "dur": 0.348,
    "args": {
      "External id": 2485416,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061761.231, "dur": 0.533,
    "args": {
      "External id": 2485417,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061763.245, "dur": 0.643,
    "args": {
      "External id": 2485418,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061766.437, "dur": 0.565,
    "args": {
      "External id": 2485419,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061768.408, "dur": 0.391,
    "args": {
      "External id": 2485420,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061770.551, "dur": 2.104,
    "args": {
      "External id": 2485421,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061773.936, "dur": 0.355,
    "args": {
      "External id": 2485422,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061776.265, "dur": 2.355,
    "args": {
      "External id": 2485423,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061781.528, "dur": 0.313,
    "args": {
      "External id": 2485424,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061794.906, "dur": 18.725,
    "args": {
      "External id": 2485425,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061863.946, "dur": 114.015,
    "args": {
      "External id": 2485426,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061892.268, "dur": 82.722,
    "args": {
      "External id": 2485427,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2226, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657061901.585, "dur": 69.145,
    "args": {
      "External id": 2485428,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062032.662, "dur": 2.999,
    "args": {
      "External id": 2485429,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2228, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062117.227, "dur": 1660.425,
    "args": {
      "External id": 2485430,"Sequence number": 24740468, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2229
    }
  },
  {
    "ph": "f", "id": 180, "pid": 1336757, "tid": 1381170, "ts": 1514657062117.227,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062227.237, "dur": 105.073,
    "args": {
      "External id": 2485431,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062372.863, "dur": 41.092,
    "args": {
      "External id": 2485432,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062431.927, "dur": 48.733,
    "args": {
      "External id": 2485433,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062490.079, "dur": 31.599,
    "args": {
      "External id": 2485434,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062544.067, "dur": 52.007,
    "args": {
      "External id": 2485435,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062606.782, "dur": 28.511,
    "args": {
      "External id": 2485436,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062645.487, "dur": 42.423,
    "args": {
      "External id": 2485437,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062714.343, "dur": 26.524,
    "args": {
      "External id": 2485438,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062759.659, "dur": 29.619,
    "args": {
      "External id": 2485439,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062812.072, "dur": 20.577,
    "args": {
      "External id": 2485440,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062847.013, "dur": 16.754,
    "args": {
      "External id": 2485441,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062875.411, "dur": 29.580,
    "args": {
      "External id": 2485442,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062908.074, "dur": 32.905,
    "args": {
      "External id": 2485443,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657062966.861, "dur": 212.597,
    "args": {
      "External id": 2485444,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063082.106, "dur": 7.292,
    "args": {
      "External id": 2485445,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063091.447, "dur": 2.284,
    "args": {
      "External id": 2485446,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063217.857, "dur": 24.743,
    "args": {
      "External id": 2485447,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063256.550, "dur": 15.904,
    "args": {
      "External id": 2485448,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063282.614, "dur": 40.440,
    "args": {
      "External id": 2485449,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063328.417, "dur": 35.985,
    "args": {
      "External id": 2485450,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063378.901, "dur": 34.152,
    "args": {
      "External id": 2485451,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063423.164, "dur": 37.961,
    "args": {
      "External id": 2485452,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063469.696, "dur": 21.444,
    "args": {
      "External id": 2485453,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063498.630, "dur": 47.598,
    "args": {
      "External id": 2485454,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063572.681, "dur": 29.331,
    "args": {
      "External id": 2485455,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063621.032, "dur": 27.406,
    "args": {
      "External id": 2485456,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063666.590, "dur": 17.049,
    "args": {
      "External id": 2485457,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063701.635, "dur": 16.179,
    "args": {
      "External id": 2485458,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063729.667, "dur": 18.042,
    "args": {
      "External id": 2485459,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063822.831, "dur": 14.775,
    "args": {
      "External id": 2485460,"Record function id": 0, "Ev Idx": 2259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063826.014, "dur": 10.646,
    "args": {
      "External id": 2485461,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063829.970, "dur": 5.762,
    "args": {
      "External id": 2485462,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063831.591, "dur": 3.991,
    "args": {
      "External id": 2485463,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063841.514, "dur": 6.011,
    "args": {
      "External id": 2485464,"Record function id": 0, "Ev Idx": 2263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063843.327, "dur": 3.751,
    "args": {
      "External id": 2485465,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063844.460, "dur": 2.004,
    "args": {
      "External id": 2485466,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063845.467, "dur": 0.911,
    "args": {
      "External id": 2485467,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063850.751, "dur": 4.682,
    "args": {
      "External id": 2485468,"Record function id": 0, "Ev Idx": 2267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063852.303, "dur": 2.683,
    "args": {
      "External id": 2485469,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063853.073, "dur": 1.456,
    "args": {
      "External id": 2485470,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063853.434, "dur": 1.002,
    "args": {
      "External id": 2485471,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063858.563, "dur": 4.231,
    "args": {
      "External id": 2485472,"Record function id": 0, "Ev Idx": 2271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063859.793, "dur": 2.593,
    "args": {
      "External id": 2485473,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063860.532, "dur": 1.319,
    "args": {
      "External id": 2485474,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063861.126, "dur": 0.654,
    "args": {
      "External id": 2485475,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063865.848, "dur": 3.806,
    "args": {
      "External id": 2485476,"Record function id": 0, "Ev Idx": 2275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063867.159, "dur": 2.074,
    "args": {
      "External id": 2485477,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063867.608, "dur": 1.097,
    "args": {
      "External id": 2485478,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063868.064, "dur": 0.572,
    "args": {
      "External id": 2485479,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063872.779, "dur": 3.863,
    "args": {
      "External id": 2485480,"Record function id": 0, "Ev Idx": 2279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063873.858, "dur": 2.368,
    "args": {
      "External id": 2485481,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063874.457, "dur": 1.143,
    "args": {
      "External id": 2485482,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063874.904, "dur": 0.601,
    "args": {
      "External id": 2485483,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063880.066, "dur": 5.382,
    "args": {
      "External id": 2485484,"Record function id": 0, "Ev Idx": 2283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063881.172, "dur": 3.851,
    "args": {
      "External id": 2485485,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063881.673, "dur": 2.816,
    "args": {
      "External id": 2485486,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063881.998, "dur": 2.399,
    "args": {
      "External id": 2485487,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063888.591, "dur": 3.986,
    "args": {
      "External id": 2485488,"Record function id": 0, "Ev Idx": 2287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063889.658, "dur": 2.493,
    "args": {
      "External id": 2485489,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063890.311, "dur": 1.277,
    "args": {
      "External id": 2485490,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063890.747, "dur": 0.744,
    "args": {
      "External id": 2485491,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063895.684, "dur": 4.012,
    "args": {
      "External id": 2485492,"Record function id": 0, "Ev Idx": 2291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063896.774, "dur": 2.483,
    "args": {
      "External id": 2485493,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063897.244, "dur": 1.399,
    "args": {
      "External id": 2485494,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063897.865, "dur": 0.671,
    "args": {
      "External id": 2485495,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063903.318, "dur": 38943.483,
    "args": {
      "External id": 2485496,"Record function id": 0, "Sequence number": 24740467, "Fwd thread id": 1, "Ev Idx": 2295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063904.577, "dur": 38933.531,
    "args": {
      "External id": 2485497,"Sequence number": 24740467, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2296
    }
  },
  {
    "ph": "f", "id": 181, "pid": 1336757, "tid": 1381170, "ts": 1514657063904.577,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.23)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063933.935, "dur": 35.837,
    "args": {
      "External id": 2485498,"Record function id": 0, "Ev Idx": 2297
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.23)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657063977.526, "dur": 102.545,
    "args": {
      "External id": 2485499,"Record function id": 0, "Ev Idx": 2298
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.23)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657064087.892, "dur": 38743.153,
    "args": {
      "External id": 2485500,"Record function id": 0, "Ev Idx": 2299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657064175.160, "dur": 6.738,
    "args": {
      "External id": 2485501,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657064192.967, "dur": 5.199,
    "args": {
      "External id": 2485502,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657064212.414, "dur": 37843.128,
    "args": {
      "External id": 2485503,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657064226.065, "dur": 37820.956,
    "args": {
      "External id": 2485504,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657064273.367, "dur": 13.661,
    "args": {
      "External id": 2485505,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657064293.455, "dur": 37713.384,
    "args": {
      "External id": 2485506,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657064296.004, "dur": 37709.930,
    "args": {
      "External id": 2485507,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657064300.307, "dur": 4.901,
    "args": {
      "External id": 2485508,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657064307.015, "dur": 37694.467,
    "args": {
      "External id": 2485509,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102141.773, "dur": 9.486,
    "args": {
      "External id": 2485510,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102145.134, "dur": 5.813,
    "args": {
      "External id": 2485511,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102182.611, "dur": 331.872,
    "args": {
      "External id": 2485512,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102209.089, "dur": 301.050,
    "args": {
      "External id": 2485513,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2312, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102220.197, "dur": 283.782,
    "args": {
      "External id": 2485514,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102549.406, "dur": 3.408,
    "args": {
      "External id": 2485515,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2314, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102611.445, "dur": 7.952,
    "args": {
      "External id": 2485516,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102666.252, "dur": 1.656,
    "args": {
      "External id": 2485517,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102684.946, "dur": 0.942,
    "args": {
      "External id": 2485518,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102698.362, "dur": 0.828,
    "args": {
      "External id": 2485519,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102711.444, "dur": 2.675,
    "args": {
      "External id": 2485520,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102727.079, "dur": 0.806,
    "args": {
      "External id": 2485521,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102740.411, "dur": 1.065,
    "args": {
      "External id": 2485522,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102753.993, "dur": 2.011,
    "args": {
      "External id": 2485523,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102767.417, "dur": 2.427,
    "args": {
      "External id": 2485524,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102862.422, "dur": 2804.961,
    "args": {
      "External id": 2485525,"Record function id": 0, "Ev Idx": 2324
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.22)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102881.082, "dur": 1040.548,
    "args": {
      "External id": 2485526,"Record function id": 0, "Ev Idx": 2325
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.22)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102895.292, "dur": 358.071,
    "args": {
      "External id": 2485527,"Record function id": 0, "Ev Idx": 2326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657102977.439, "dur": 35.691,
    "args": {
      "External id": 2485528,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103019.060, "dur": 2.057,
    "args": {
      "External id": 2485529,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103023.183, "dur": 1.166,
    "args": {
      "External id": 2485530,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103026.163, "dur": 0.835,
    "args": {
      "External id": 2485531,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103028.575, "dur": 1.098,
    "args": {
      "External id": 2485532,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103031.248, "dur": 1.065,
    "args": {
      "External id": 2485533,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103033.927, "dur": 4.021,
    "args": {
      "External id": 2485534,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103039.756, "dur": 1.123,
    "args": {
      "External id": 2485535,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103042.597, "dur": 0.846,
    "args": {
      "External id": 2485536,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103044.994, "dur": 0.925,
    "args": {
      "External id": 2485537,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103065.095, "dur": 155.821,
    "args": {
      "External id": 2485538,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103081.148, "dur": 134.841,
    "args": {
      "External id": 2485539,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103104.194, "dur": 12.084,
    "args": {
      "External id": 2485540,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103119.678, "dur": 68.069,
    "args": {
      "External id": 2485541,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103122.295, "dur": 65.039,
    "args": {
      "External id": 2485542,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103126.833, "dur": 5.833,
    "args": {
      "External id": 2485543,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103134.337, "dur": 52.437,
    "args": {
      "External id": 2485544,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2343
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.21", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103344.963, "dur": 568.622,
    "args": {
      "External id": 2485545,"Record function id": 0, "Ev Idx": 2344
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.21)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103360.496, "dur": 541.284,
    "args": {
      "External id": 2485546,"Record function id": 0, "Ev Idx": 2345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103424.326, "dur": 4.310,
    "args": {
      "External id": 2485547,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103443.812, "dur": 32.176,
    "args": {
      "External id": 2485548,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103448.827, "dur": 1.564,
    "args": {
      "External id": 2485549,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103452.550, "dur": 2.642,
    "args": {
      "External id": 2485550,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103457.178, "dur": 0.360,
    "args": {
      "External id": 2485551,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103459.395, "dur": 0.555,
    "args": {
      "External id": 2485552,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103461.989, "dur": 0.345,
    "args": {
      "External id": 2485553,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103464.149, "dur": 0.598,
    "args": {
      "External id": 2485554,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103466.877, "dur": 0.440,
    "args": {
      "External id": 2485555,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103469.073, "dur": 0.509,
    "args": {
      "External id": 2485556,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103471.331, "dur": 0.390,
    "args": {
      "External id": 2485557,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103490.878, "dur": 32.191,
    "args": {
      "External id": 2485558,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103575.640, "dur": 110.725,
    "args": {
      "External id": 2485559,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103587.453, "dur": 5.995,
    "args": {
      "External id": 2485560,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103598.882, "dur": 11.095,
    "args": {
      "External id": 2485561,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103603.071, "dur": 6.461,
    "args": {
      "External id": 2485562,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103607.328, "dur": 0.647,
    "args": {
      "External id": 2485563,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103617.326, "dur": 27.225,
    "args": {
      "External id": 2485564,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103619.957, "dur": 0.329,
    "args": {
      "External id": 2485565,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103622.394, "dur": 0.596,
    "args": {
      "External id": 2485566,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103624.721, "dur": 0.552,
    "args": {
      "External id": 2485567,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103626.898, "dur": 0.356,
    "args": {
      "External id": 2485568,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103629.046, "dur": 0.357,
    "args": {
      "External id": 2485569,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103630.978, "dur": 0.406,
    "args": {
      "External id": 2485570,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103633.247, "dur": 2.185,
    "args": {
      "External id": 2485571,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103637.168, "dur": 0.393,
    "args": {
      "External id": 2485572,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103639.399, "dur": 0.523,
    "args": {
      "External id": 2485573,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103656.835, "dur": 21.519,
    "args": {
      "External id": 2485574,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103731.818, "dur": 107.224,
    "args": {
      "External id": 2485575,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103754.129, "dur": 81.832,
    "args": {
      "External id": 2485576,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2375, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103764.117, "dur": 67.727,
    "args": {
      "External id": 2485577,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103854.389, "dur": 2.070,
    "args": {
      "External id": 2485578,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2377, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657103929.460, "dur": 1715.124,
    "args": {
      "External id": 2485579,"Sequence number": 24740466, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2378
    }
  },
  {
    "ph": "f", "id": 182, "pid": 1336757, "tid": 1381170, "ts": 1514657103929.460,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104078.265, "dur": 106.522,
    "args": {
      "External id": 2485580,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104220.225, "dur": 39.762,
    "args": {
      "External id": 2485581,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104278.802, "dur": 50.519,
    "args": {
      "External id": 2485582,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104339.167, "dur": 33.436,
    "args": {
      "External id": 2485583,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104380.618, "dur": 47.354,
    "args": {
      "External id": 2485584,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104438.499, "dur": 30.103,
    "args": {
      "External id": 2485585,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104477.486, "dur": 44.113,
    "args": {
      "External id": 2485586,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104561.417, "dur": 26.240,
    "args": {
      "External id": 2485587,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104617.661, "dur": 30.204,
    "args": {
      "External id": 2485588,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104671.492, "dur": 19.153,
    "args": {
      "External id": 2485589,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104704.798, "dur": 17.600,
    "args": {
      "External id": 2485590,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104735.547, "dur": 33.865,
    "args": {
      "External id": 2485591,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104772.828, "dur": 35.048,
    "args": {
      "External id": 2485592,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104836.644, "dur": 212.470,
    "args": {
      "External id": 2485593,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104914.805, "dur": 6.192,
    "args": {
      "External id": 2485594,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657104922.991, "dur": 4.420,
    "args": {
      "External id": 2485595,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105084.793, "dur": 26.911,
    "args": {
      "External id": 2485596,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105124.018, "dur": 17.743,
    "args": {
      "External id": 2485597,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105152.007, "dur": 40.558,
    "args": {
      "External id": 2485598,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105199.573, "dur": 36.614,
    "args": {
      "External id": 2485599,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105243.615, "dur": 22.358,
    "args": {
      "External id": 2485600,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105273.713, "dur": 48.158,
    "args": {
      "External id": 2485601,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105334.731, "dur": 24.024,
    "args": {
      "External id": 2485602,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105367.854, "dur": 35.636,
    "args": {
      "External id": 2485603,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105421.983, "dur": 23.387,
    "args": {
      "External id": 2485604,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105463.636, "dur": 25.151,
    "args": {
      "External id": 2485605,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105504.134, "dur": 18.810,
    "args": {
      "External id": 2485606,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105555.765, "dur": 19.018,
    "args": {
      "External id": 2485607,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105597.646, "dur": 18.778,
    "args": {
      "External id": 2485608,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105690.275, "dur": 15.479,
    "args": {
      "External id": 2485609,"Record function id": 0, "Ev Idx": 2408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105694.126, "dur": 10.788,
    "args": {
      "External id": 2485610,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105698.163, "dur": 5.857,
    "args": {
      "External id": 2485611,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105699.905, "dur": 3.968,
    "args": {
      "External id": 2485612,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105709.581, "dur": 6.024,
    "args": {
      "External id": 2485613,"Record function id": 0, "Ev Idx": 2412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105711.597, "dur": 3.563,
    "args": {
      "External id": 2485614,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105712.429, "dur": 2.220,
    "args": {
      "External id": 2485615,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105713.388, "dur": 1.181,
    "args": {
      "External id": 2485616,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105718.930, "dur": 4.233,
    "args": {
      "External id": 2485617,"Record function id": 0, "Ev Idx": 2416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105720.343, "dur": 2.387,
    "args": {
      "External id": 2485618,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105720.912, "dur": 1.433,
    "args": {
      "External id": 2485619,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105721.219, "dur": 1.046,
    "args": {
      "External id": 2485620,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105726.339, "dur": 5.803,
    "args": {
      "External id": 2485621,"Record function id": 0, "Ev Idx": 2420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105727.599, "dur": 4.126,
    "args": {
      "External id": 2485622,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105728.110, "dur": 3.042,
    "args": {
      "External id": 2485623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105728.413, "dur": 2.659,
    "args": {
      "External id": 2485624,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105735.231, "dur": 3.770,
    "args": {
      "External id": 2485625,"Record function id": 0, "Ev Idx": 2424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105736.268, "dur": 2.309,
    "args": {
      "External id": 2485626,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105736.917, "dur": 1.238,
    "args": {
      "External id": 2485627,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105737.239, "dur": 0.851,
    "args": {
      "External id": 2485628,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105742.462, "dur": 3.773,
    "args": {
      "External id": 2485629,"Record function id": 0, "Ev Idx": 2428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105743.782, "dur": 2.051,
    "args": {
      "External id": 2485630,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105744.250, "dur": 1.159,
    "args": {
      "External id": 2485631,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105744.671, "dur": 0.637,
    "args": {
      "External id": 2485632,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105749.735, "dur": 3.716,
    "args": {
      "External id": 2485633,"Record function id": 0, "Ev Idx": 2432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105751.130, "dur": 1.920,
    "args": {
      "External id": 2485634,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105751.721, "dur": 0.929,
    "args": {
      "External id": 2485635,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105752.012, "dur": 0.529,
    "args": {
      "External id": 2485636,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105757.155, "dur": 3.597,
    "args": {
      "External id": 2485637,"Record function id": 0, "Ev Idx": 2436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105758.265, "dur": 2.058,
    "args": {
      "External id": 2485638,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105758.893, "dur": 1.040,
    "args": {
      "External id": 2485639,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105759.236, "dur": 0.632,
    "args": {
      "External id": 2485640,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105764.123, "dur": 4.009,
    "args": {
      "External id": 2485641,"Record function id": 0, "Ev Idx": 2440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105765.386, "dur": 2.330,
    "args": {
      "External id": 2485642,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105765.874, "dur": 1.316,
    "args": {
      "External id": 2485643,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105766.319, "dur": 0.784,
    "args": {
      "External id": 2485644,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105772.441, "dur": 38363.706,
    "args": {
      "External id": 2485645,"Record function id": 0, "Sequence number": 24740465, "Fwd thread id": 1, "Ev Idx": 2444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105774.255, "dur": 38353.370,
    "args": {
      "External id": 2485646,"Sequence number": 24740465, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2445
    }
  },
  {
    "ph": "f", "id": 183, "pid": 1336757, "tid": 1381170, "ts": 1514657105774.255,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.22)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105803.922, "dur": 36.857,
    "args": {
      "External id": 2485647,"Record function id": 0, "Ev Idx": 2446
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.22)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105848.594, "dur": 66.477,
    "args": {
      "External id": 2485648,"Record function id": 0, "Ev Idx": 2447
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.22)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657105921.353, "dur": 38198.214,
    "args": {
      "External id": 2485649,"Record function id": 0, "Ev Idx": 2448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657106042.178, "dur": 7.732,
    "args": {
      "External id": 2485650,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657106060.466, "dur": 4.931,
    "args": {
      "External id": 2485651,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657106080.707, "dur": 37202.768,
    "args": {
      "External id": 2485652,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657106094.114, "dur": 37180.611,
    "args": {
      "External id": 2485653,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657106146.727, "dur": 16.151,
    "args": {
      "External id": 2485654,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657106169.600, "dur": 37065.177,
    "args": {
      "External id": 2485655,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657106172.347, "dur": 37061.656,
    "args": {
      "External id": 2485656,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657106176.614, "dur": 4.851,
    "args": {
      "External id": 2485657,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657106183.240, "dur": 37047.106,
    "args": {
      "External id": 2485658,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657143371.132, "dur": 8.795,
    "args": {
      "External id": 2485659,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657143374.137, "dur": 5.438,
    "args": {
      "External id": 2485660,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657143409.131, "dur": 363.764,
    "args": {
      "External id": 2485661,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657143437.111, "dur": 330.020,
    "args": {
      "External id": 2485662,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2461, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657143448.523, "dur": 312.575,
    "args": {
      "External id": 2485663,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657143794.813, "dur": 2.497,
    "args": {
      "External id": 2485664,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2463, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657143857.766, "dur": 6.280,
    "args": {
      "External id": 2485665,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657143910.534, "dur": 1.323,
    "args": {
      "External id": 2485666,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657143929.029, "dur": 1.015,
    "args": {
      "External id": 2485667,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657143946.119, "dur": 1.376,
    "args": {
      "External id": 2485668,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657143959.389, "dur": 1.027,
    "args": {
      "External id": 2485669,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657143973.140, "dur": 0.889,
    "args": {
      "External id": 2485670,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144021.761, "dur": 1.858,
    "args": {
      "External id": 2485671,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144040.933, "dur": 2.776,
    "args": {
      "External id": 2485672,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144055.714, "dur": 0.856,
    "args": {
      "External id": 2485673,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144153.083, "dur": 2871.046,
    "args": {
      "External id": 2485674,"Record function id": 0, "Ev Idx": 2473
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.21)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144173.778, "dur": 1081.514,
    "args": {
      "External id": 2485675,"Record function id": 0, "Ev Idx": 2474
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.21)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144190.793, "dur": 352.480,
    "args": {
      "External id": 2485676,"Record function id": 0, "Ev Idx": 2475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144277.573, "dur": 4.070,
    "args": {
      "External id": 2485677,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144284.799, "dur": 1.233,
    "args": {
      "External id": 2485678,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144288.283, "dur": 1.204,
    "args": {
      "External id": 2485679,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144291.427, "dur": 3.144,
    "args": {
      "External id": 2485680,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144296.125, "dur": 1.147,
    "args": {
      "External id": 2485681,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144298.892, "dur": 1.309,
    "args": {
      "External id": 2485682,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144302.157, "dur": 1.975,
    "args": {
      "External id": 2485683,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144305.941, "dur": 1.057,
    "args": {
      "External id": 2485684,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144308.546, "dur": 0.995,
    "args": {
      "External id": 2485685,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144311.007, "dur": 0.897,
    "args": {
      "External id": 2485686,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144330.099, "dur": 165.217,
    "args": {
      "External id": 2485687,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144359.941, "dur": 131.235,
    "args": {
      "External id": 2485688,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144377.837, "dur": 12.356,
    "args": {
      "External id": 2485689,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144393.598, "dur": 69.004,
    "args": {
      "External id": 2485690,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144396.302, "dur": 65.907,
    "args": {
      "External id": 2485691,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144400.537, "dur": 7.718,
    "args": {
      "External id": 2485692,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144409.984, "dur": 51.434,
    "args": {
      "External id": 2485693,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2492
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.20", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144641.394, "dur": 605.319,
    "args": {
      "External id": 2485694,"Record function id": 0, "Ev Idx": 2493
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.20)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144659.371, "dur": 574.421,
    "args": {
      "External id": 2485695,"Record function id": 0, "Ev Idx": 2494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144724.054, "dur": 5.741,
    "args": {
      "External id": 2485696,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144745.920, "dur": 33.695,
    "args": {
      "External id": 2485697,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144750.899, "dur": 1.454,
    "args": {
      "External id": 2485698,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144754.774, "dur": 0.486,
    "args": {
      "External id": 2485699,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144757.315, "dur": 0.449,
    "args": {
      "External id": 2485700,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144759.578, "dur": 0.665,
    "args": {
      "External id": 2485701,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144761.990, "dur": 0.605,
    "args": {
      "External id": 2485702,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144764.136, "dur": 0.440,
    "args": {
      "External id": 2485703,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144766.523, "dur": 2.706,
    "args": {
      "External id": 2485704,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144770.872, "dur": 0.743,
    "args": {
      "External id": 2485705,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144773.171, "dur": 0.627,
    "args": {
      "External id": 2485706,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144790.175, "dur": 31.823,
    "args": {
      "External id": 2485707,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144854.352, "dur": 100.451,
    "args": {
      "External id": 2485708,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144864.666, "dur": 3.528,
    "args": {
      "External id": 2485709,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144873.191, "dur": 10.283,
    "args": {
      "External id": 2485710,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144877.452, "dur": 5.592,
    "args": {
      "External id": 2485711,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144881.307, "dur": 0.497,
    "args": {
      "External id": 2485712,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144890.256, "dur": 29.276,
    "args": {
      "External id": 2485713,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144892.797, "dur": 0.591,
    "args": {
      "External id": 2485714,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144895.062, "dur": 0.434,
    "args": {
      "External id": 2485715,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144897.420, "dur": 0.670,
    "args": {
      "External id": 2485716,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144899.929, "dur": 2.287,
    "args": {
      "External id": 2485717,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144904.388, "dur": 0.813,
    "args": {
      "External id": 2485718,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144907.104, "dur": 0.393,
    "args": {
      "External id": 2485719,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144909.329, "dur": 0.426,
    "args": {
      "External id": 2485720,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144911.745, "dur": 0.415,
    "args": {
      "External id": 2485721,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144913.863, "dur": 0.512,
    "args": {
      "External id": 2485722,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657144929.141, "dur": 18.303,
    "args": {
      "External id": 2485723,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657145050.009, "dur": 116.207,
    "args": {
      "External id": 2485724,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657145073.158, "dur": 89.640,
    "args": {
      "External id": 2485725,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2524, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657145084.047, "dur": 74.255,
    "args": {
      "External id": 2485726,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657145179.482, "dur": 2.159,
    "args": {
      "External id": 2485727,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2526, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657145262.475, "dur": 1704.606,
    "args": {
      "External id": 2485728,"Sequence number": 24740464, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2527
    }
  },
  {
    "ph": "f", "id": 184, "pid": 1336757, "tid": 1381170, "ts": 1514657145262.475,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657145371.839, "dur": 102.228,
    "args": {
      "External id": 2485729,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657145513.006, "dur": 59.747,
    "args": {
      "External id": 2485730,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657145594.957, "dur": 56.837,
    "args": {
      "External id": 2485731,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657145662.226, "dur": 35.056,
    "args": {
      "External id": 2485732,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657145707.077, "dur": 47.780,
    "args": {
      "External id": 2485733,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657145764.180, "dur": 29.472,
    "args": {
      "External id": 2485734,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657145801.932, "dur": 43.688,
    "args": {
      "External id": 2485735,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657145870.576, "dur": 24.637,
    "args": {
      "External id": 2485736,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657145916.077, "dur": 30.571,
    "args": {
      "External id": 2485737,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657145968.196, "dur": 55.972,
    "args": {
      "External id": 2485738,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146042.046, "dur": 18.924,
    "args": {
      "External id": 2485739,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146072.950, "dur": 37.625,
    "args": {
      "External id": 2485740,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146114.415, "dur": 37.442,
    "args": {
      "External id": 2485741,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146182.254, "dur": 172.409,
    "args": {
      "External id": 2485742,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146258.484, "dur": 7.061,
    "args": {
      "External id": 2485743,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146267.562, "dur": 2.675,
    "args": {
      "External id": 2485744,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146414.734, "dur": 25.305,
    "args": {
      "External id": 2485745,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146453.102, "dur": 17.030,
    "args": {
      "External id": 2485746,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146479.343, "dur": 35.414,
    "args": {
      "External id": 2485747,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146522.124, "dur": 65.876,
    "args": {
      "External id": 2485748,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146610.483, "dur": 33.334,
    "args": {
      "External id": 2485749,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146649.411, "dur": 33.148,
    "args": {
      "External id": 2485750,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146689.972, "dur": 24.614,
    "args": {
      "External id": 2485751,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146722.595, "dur": 31.437,
    "args": {
      "External id": 2485752,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146777.412, "dur": 24.146,
    "args": {
      "External id": 2485753,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146818.095, "dur": 25.139,
    "args": {
      "External id": 2485754,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146857.464, "dur": 18.467,
    "args": {
      "External id": 2485755,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146892.398, "dur": 16.441,
    "args": {
      "External id": 2485756,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657146921.514, "dur": 17.596,
    "args": {
      "External id": 2485757,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147049.521, "dur": 43.322,
    "args": {
      "External id": 2485758,"Record function id": 0, "Ev Idx": 2557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147053.165, "dur": 38.429,
    "args": {
      "External id": 2485759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147057.485, "dur": 32.607,
    "args": {
      "External id": 2485760,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147084.884, "dur": 5.070,
    "args": {
      "External id": 2485761,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147097.371, "dur": 5.097,
    "args": {
      "External id": 2485762,"Record function id": 0, "Ev Idx": 2561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147098.820, "dur": 3.208,
    "args": {
      "External id": 2485763,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147099.438, "dur": 1.817,
    "args": {
      "External id": 2485764,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147100.130, "dur": 1.024,
    "args": {
      "External id": 2485765,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147105.952, "dur": 6.515,
    "args": {
      "External id": 2485766,"Record function id": 0, "Ev Idx": 2565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147107.362, "dur": 4.706,
    "args": {
      "External id": 2485767,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147107.888, "dur": 3.653,
    "args": {
      "External id": 2485768,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147108.274, "dur": 3.200,
    "args": {
      "External id": 2485769,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147115.846, "dur": 3.577,
    "args": {
      "External id": 2485770,"Record function id": 0, "Ev Idx": 2569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147116.968, "dur": 2.037,
    "args": {
      "External id": 2485771,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147117.507, "dur": 1.086,
    "args": {
      "External id": 2485772,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147117.826, "dur": 0.702,
    "args": {
      "External id": 2485773,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147122.762, "dur": 3.858,
    "args": {
      "External id": 2485774,"Record function id": 0, "Ev Idx": 2573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147123.957, "dur": 2.226,
    "args": {
      "External id": 2485775,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147124.621, "dur": 0.915,
    "args": {
      "External id": 2485776,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147124.910, "dur": 0.562,
    "args": {
      "External id": 2485777,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147130.227, "dur": 4.153,
    "args": {
      "External id": 2485778,"Record function id": 0, "Ev Idx": 2577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147131.599, "dur": 2.374,
    "args": {
      "External id": 2485779,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147132.072, "dur": 1.298,
    "args": {
      "External id": 2485780,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147132.529, "dur": 0.771,
    "args": {
      "External id": 2485781,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147137.918, "dur": 3.737,
    "args": {
      "External id": 2485782,"Record function id": 0, "Ev Idx": 2581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147139.132, "dur": 2.097,
    "args": {
      "External id": 2485783,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147139.602, "dur": 0.994,
    "args": {
      "External id": 2485784,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147139.908, "dur": 0.608,
    "args": {
      "External id": 2485785,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147144.918, "dur": 5.662,
    "args": {
      "External id": 2485786,"Record function id": 0, "Ev Idx": 2585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147146.439, "dur": 3.744,
    "args": {
      "External id": 2485787,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147146.906, "dur": 2.837,
    "args": {
      "External id": 2485788,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147148.963, "dur": 0.673,
    "args": {
      "External id": 2485789,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147153.940, "dur": 3.869,
    "args": {
      "External id": 2485790,"Record function id": 0, "Ev Idx": 2589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147155.338, "dur": 2.035,
    "args": {
      "External id": 2485791,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147155.818, "dur": 0.967,
    "args": {
      "External id": 2485792,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147156.088, "dur": 0.598,
    "args": {
      "External id": 2485793,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147162.363, "dur": 39862.059,
    "args": {
      "External id": 2485794,"Record function id": 0, "Sequence number": 24740463, "Fwd thread id": 1, "Ev Idx": 2593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147163.695, "dur": 39849.672,
    "args": {
      "External id": 2485795,"Sequence number": 24740463, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2594
    }
  },
  {
    "ph": "f", "id": 185, "pid": 1336757, "tid": 1381170, "ts": 1514657147163.695,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.21)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147194.955, "dur": 37.496,
    "args": {
      "External id": 2485796,"Record function id": 0, "Ev Idx": 2595
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.21)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147240.211, "dur": 69.577,
    "args": {
      "External id": 2485797,"Record function id": 0, "Ev Idx": 2596
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.21)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147315.606, "dur": 39657.977,
    "args": {
      "External id": 2485798,"Record function id": 0, "Ev Idx": 2597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147404.948, "dur": 7.547,
    "args": {
      "External id": 2485799,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147422.001, "dur": 6.797,
    "args": {
      "External id": 2485800,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147447.811, "dur": 38655.709,
    "args": {
      "External id": 2485801,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147461.320, "dur": 38630.408,
    "args": {
      "External id": 2485802,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147508.253, "dur": 13.619,
    "args": {
      "External id": 2485803,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147544.603, "dur": 38502.670,
    "args": {
      "External id": 2485804,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147547.313, "dur": 38499.124,
    "args": {
      "External id": 2485805,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147551.779, "dur": 7.007,
    "args": {
      "External id": 2485806,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657147560.939, "dur": 38481.119,
    "args": {
      "External id": 2485807,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657186208.400, "dur": 11.370,
    "args": {
      "External id": 2485808,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657186212.129, "dur": 7.308,
    "args": {
      "External id": 2485809,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657186250.939, "dur": 407.253,
    "args": {
      "External id": 2485810,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657186278.332, "dur": 374.520,
    "args": {
      "External id": 2485811,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2610, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657186290.047, "dur": 356.398,
    "args": {
      "External id": 2485812,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657186680.488, "dur": 2.838,
    "args": {
      "External id": 2485813,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2612, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657186746.918, "dur": 6.691,
    "args": {
      "External id": 2485814,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657186801.454, "dur": 2.270,
    "args": {
      "External id": 2485815,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657186822.015, "dur": 3.490,
    "args": {
      "External id": 2485816,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657186839.721, "dur": 1.203,
    "args": {
      "External id": 2485817,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657186854.524, "dur": 1.120,
    "args": {
      "External id": 2485818,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657186867.792, "dur": 1.056,
    "args": {
      "External id": 2485819,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657186880.723, "dur": 3.196,
    "args": {
      "External id": 2485820,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657186896.331, "dur": 2.517,
    "args": {
      "External id": 2485821,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657186910.881, "dur": 0.943,
    "args": {
      "External id": 2485822,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187042.448, "dur": 2812.013,
    "args": {
      "External id": 2485823,"Record function id": 0, "Ev Idx": 2622
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.20)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187064.926, "dur": 1074.861,
    "args": {
      "External id": 2485824,"Record function id": 0, "Ev Idx": 2623
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.20)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187079.537, "dur": 339.399,
    "args": {
      "External id": 2485825,"Record function id": 0, "Ev Idx": 2624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187168.339, "dur": 4.587,
    "args": {
      "External id": 2485826,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187176.188, "dur": 1.346,
    "args": {
      "External id": 2485827,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187179.315, "dur": 2.871,
    "args": {
      "External id": 2485828,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187183.964, "dur": 1.086,
    "args": {
      "External id": 2485829,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187186.767, "dur": 1.031,
    "args": {
      "External id": 2485830,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187189.814, "dur": 1.313,
    "args": {
      "External id": 2485831,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187192.961, "dur": 2.657,
    "args": {
      "External id": 2485832,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187197.235, "dur": 1.243,
    "args": {
      "External id": 2485833,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187199.967, "dur": 1.272,
    "args": {
      "External id": 2485834,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187203.020, "dur": 0.999,
    "args": {
      "External id": 2485835,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187223.732, "dur": 162.534,
    "args": {
      "External id": 2485836,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187240.860, "dur": 141.278,
    "args": {
      "External id": 2485837,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187266.721, "dur": 13.934,
    "args": {
      "External id": 2485838,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187284.149, "dur": 68.287,
    "args": {
      "External id": 2485839,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187286.848, "dur": 65.216,
    "args": {
      "External id": 2485840,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187291.376, "dur": 5.608,
    "args": {
      "External id": 2485841,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187299.068, "dur": 52.502,
    "args": {
      "External id": 2485842,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2641
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.19", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187509.122, "dur": 622.586,
    "args": {
      "External id": 2485843,"Record function id": 0, "Ev Idx": 2642
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.19)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187524.478, "dur": 593.699,
    "args": {
      "External id": 2485844,"Record function id": 0, "Ev Idx": 2643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187608.860, "dur": 5.513,
    "args": {
      "External id": 2485845,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187630.281, "dur": 38.509,
    "args": {
      "External id": 2485846,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187635.679, "dur": 1.886,
    "args": {
      "External id": 2485847,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187640.058, "dur": 0.400,
    "args": {
      "External id": 2485848,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187642.427, "dur": 4.023,
    "args": {
      "External id": 2485849,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187648.238, "dur": 0.528,
    "args": {
      "External id": 2485850,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187651.101, "dur": 0.562,
    "args": {
      "External id": 2485851,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187653.645, "dur": 2.693,
    "args": {
      "External id": 2485852,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187657.991, "dur": 0.526,
    "args": {
      "External id": 2485853,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187660.335, "dur": 0.336,
    "args": {
      "External id": 2485854,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187662.384, "dur": 0.348,
    "args": {
      "External id": 2485855,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187679.765, "dur": 32.937,
    "args": {
      "External id": 2485856,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187744.723, "dur": 103.989,
    "args": {
      "External id": 2485857,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187755.307, "dur": 3.866,
    "args": {
      "External id": 2485858,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187764.191, "dur": 10.696,
    "args": {
      "External id": 2485859,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187768.487, "dur": 5.947,
    "args": {
      "External id": 2485860,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187772.461, "dur": 0.857,
    "args": {
      "External id": 2485861,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187781.577, "dur": 29.355,
    "args": {
      "External id": 2485862,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187784.186, "dur": 0.918,
    "args": {
      "External id": 2485863,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187786.916, "dur": 0.451,
    "args": {
      "External id": 2485864,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187789.342, "dur": 2.646,
    "args": {
      "External id": 2485865,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187793.930, "dur": 0.577,
    "args": {
      "External id": 2485866,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187796.214, "dur": 0.348,
    "args": {
      "External id": 2485867,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187798.306, "dur": 0.455,
    "args": {
      "External id": 2485868,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187800.909, "dur": 0.354,
    "args": {
      "External id": 2485869,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187803.364, "dur": 0.554,
    "args": {
      "External id": 2485870,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187805.710, "dur": 0.437,
    "args": {
      "External id": 2485871,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187822.447, "dur": 18.922,
    "args": {
      "External id": 2485872,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187896.497, "dur": 150.964,
    "args": {
      "External id": 2485873,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187918.684, "dur": 124.757,
    "args": {
      "External id": 2485874,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2673, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657187928.716, "dur": 109.887,
    "args": {
      "External id": 2485875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657188063.330, "dur": 2.016,
    "args": {
      "External id": 2485876,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2675, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657188147.384, "dur": 1686.196,
    "args": {
      "External id": 2485877,"Sequence number": 24740462, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2676
    }
  },
  {
    "ph": "f", "id": 186, "pid": 1336757, "tid": 1381170, "ts": 1514657188147.384,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657188259.857, "dur": 103.802,
    "args": {
      "External id": 2485878,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657188401.841, "dur": 42.614,
    "args": {
      "External id": 2485879,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657188462.640, "dur": 51.276,
    "args": {
      "External id": 2485880,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657188525.618, "dur": 54.470,
    "args": {
      "External id": 2485881,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657188590.774, "dur": 48.690,
    "args": {
      "External id": 2485882,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657188648.103, "dur": 29.427,
    "args": {
      "External id": 2485883,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657188685.987, "dur": 43.942,
    "args": {
      "External id": 2485884,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657188757.156, "dur": 25.691,
    "args": {
      "External id": 2485885,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657188803.978, "dur": 32.162,
    "args": {
      "External id": 2485886,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657188856.838, "dur": 20.462,
    "args": {
      "External id": 2485887,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657188890.514, "dur": 16.842,
    "args": {
      "External id": 2485888,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657188917.883, "dur": 30.579,
    "args": {
      "External id": 2485889,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657188952.033, "dur": 70.084,
    "args": {
      "External id": 2485890,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189056.468, "dur": 180.060,
    "args": {
      "External id": 2485891,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189139.125, "dur": 6.461,
    "args": {
      "External id": 2485892,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189147.903, "dur": 2.354,
    "args": {
      "External id": 2485893,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189273.144, "dur": 25.298,
    "args": {
      "External id": 2485894,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189309.832, "dur": 16.567,
    "args": {
      "External id": 2485895,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189335.482, "dur": 40.252,
    "args": {
      "External id": 2485896,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189383.264, "dur": 37.343,
    "args": {
      "External id": 2485897,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189429.579, "dur": 35.060,
    "args": {
      "External id": 2485898,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189475.781, "dur": 39.344,
    "args": {
      "External id": 2485899,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189522.927, "dur": 39.759,
    "args": {
      "External id": 2485900,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189573.920, "dur": 36.272,
    "args": {
      "External id": 2485901,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189633.281, "dur": 28.687,
    "args": {
      "External id": 2485902,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189679.689, "dur": 26.023,
    "args": {
      "External id": 2485903,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189722.978, "dur": 19.397,
    "args": {
      "External id": 2485904,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189758.138, "dur": 15.575,
    "args": {
      "External id": 2485905,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189787.474, "dur": 18.195,
    "args": {
      "External id": 2485906,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189877.143, "dur": 15.988,
    "args": {
      "External id": 2485907,"Record function id": 0, "Ev Idx": 2706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189880.821, "dur": 11.316,
    "args": {
      "External id": 2485908,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189884.662, "dur": 6.296,
    "args": {
      "External id": 2485909,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189886.489, "dur": 4.313,
    "args": {
      "External id": 2485910,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189897.498, "dur": 8.269,
    "args": {
      "External id": 2485911,"Record function id": 0, "Ev Idx": 2710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189899.454, "dur": 5.781,
    "args": {
      "External id": 2485912,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189900.215, "dur": 4.607,
    "args": {
      "External id": 2485913,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189901.186, "dur": 3.536,
    "args": {
      "External id": 2485914,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189909.443, "dur": 4.481,
    "args": {
      "External id": 2485915,"Record function id": 0, "Ev Idx": 2714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189910.870, "dur": 2.620,
    "args": {
      "External id": 2485916,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189911.602, "dur": 1.317,
    "args": {
      "External id": 2485917,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189911.920, "dur": 0.921,
    "args": {
      "External id": 2485918,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189917.295, "dur": 4.513,
    "args": {
      "External id": 2485919,"Record function id": 0, "Ev Idx": 2718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189919.022, "dur": 2.361,
    "args": {
      "External id": 2485920,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189919.693, "dur": 1.267,
    "args": {
      "External id": 2485921,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189920.260, "dur": 0.627,
    "args": {
      "External id": 2485922,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189925.105, "dur": 3.976,
    "args": {
      "External id": 2485923,"Record function id": 0, "Ev Idx": 2722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189926.412, "dur": 2.244,
    "args": {
      "External id": 2485924,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189926.889, "dur": 1.177,
    "args": {
      "External id": 2485925,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189927.324, "dur": 0.670,
    "args": {
      "External id": 2485926,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189932.636, "dur": 4.290,
    "args": {
      "External id": 2485927,"Record function id": 0, "Ev Idx": 2726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189934.026, "dur": 2.492,
    "args": {
      "External id": 2485928,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189934.714, "dur": 1.304,
    "args": {
      "External id": 2485929,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189935.246, "dur": 0.661,
    "args": {
      "External id": 2485930,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189940.538, "dur": 3.721,
    "args": {
      "External id": 2485931,"Record function id": 0, "Ev Idx": 2730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189941.851, "dur": 2.008,
    "args": {
      "External id": 2485932,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189942.345, "dur": 1.034,
    "args": {
      "External id": 2485933,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189942.650, "dur": 0.641,
    "args": {
      "External id": 2485934,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189947.406, "dur": 4.439,
    "args": {
      "External id": 2485935,"Record function id": 0, "Ev Idx": 2734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189948.715, "dur": 2.722,
    "args": {
      "External id": 2485936,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189949.349, "dur": 1.491,
    "args": {
      "External id": 2485937,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189949.840, "dur": 0.935,
    "args": {
      "External id": 2485938,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189955.131, "dur": 6.304,
    "args": {
      "External id": 2485939,"Record function id": 0, "Ev Idx": 2738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189956.409, "dur": 4.619,
    "args": {
      "External id": 2485940,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189957.345, "dur": 3.259,
    "args": {
      "External id": 2485941,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189957.795, "dur": 2.714,
    "args": {
      "External id": 2485942,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189965.334, "dur": 36819.688,
    "args": {
      "External id": 2485943,"Record function id": 0, "Sequence number": 24740461, "Fwd thread id": 1, "Ev Idx": 2742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657189966.999, "dur": 36809.309,
    "args": {
      "External id": 2485944,"Sequence number": 24740461, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2743
    }
  },
  {
    "ph": "f", "id": 187, "pid": 1336757, "tid": 1381170, "ts": 1514657189966.999,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.20)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657190030.486, "dur": 37.837,
    "args": {
      "External id": 2485945,"Record function id": 0, "Ev Idx": 2744
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.20)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657190076.627, "dur": 66.594,
    "args": {
      "External id": 2485946,"Record function id": 0, "Ev Idx": 2745
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.20)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657190148.508, "dur": 36620.055,
    "args": {
      "External id": 2485947,"Record function id": 0, "Ev Idx": 2746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657190236.179, "dur": 6.839,
    "args": {
      "External id": 2485948,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657190253.500, "dur": 4.894,
    "args": {
      "External id": 2485949,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657190272.869, "dur": 35651.783,
    "args": {
      "External id": 2485950,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657190286.420, "dur": 35629.228,
    "args": {
      "External id": 2485951,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657190376.051, "dur": 14.470,
    "args": {
      "External id": 2485952,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657190396.926, "dur": 35480.316,
    "args": {
      "External id": 2485953,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657190399.724, "dur": 35476.717,
    "args": {
      "External id": 2485954,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657190403.972, "dur": 5.504,
    "args": {
      "External id": 2485955,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657190411.276, "dur": 35460.533,
    "args": {
      "External id": 2485956,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226042.731, "dur": 10.185,
    "args": {
      "External id": 2485957,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226046.091, "dur": 6.288,
    "args": {
      "External id": 2485958,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226088.259, "dur": 354.800,
    "args": {
      "External id": 2485959,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226119.371, "dur": 318.198,
    "args": {
      "External id": 2485960,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2759, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226131.008, "dur": 300.301,
    "args": {
      "External id": 2485961,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226463.940, "dur": 2.370,
    "args": {
      "External id": 2485962,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2761, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226526.878, "dur": 20.487,
    "args": {
      "External id": 2485963,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226596.454, "dur": 3.720,
    "args": {
      "External id": 2485964,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226618.888, "dur": 1.164,
    "args": {
      "External id": 2485965,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226634.559, "dur": 0.866,
    "args": {
      "External id": 2485966,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226649.394, "dur": 1.248,
    "args": {
      "External id": 2485967,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226662.991, "dur": 2.978,
    "args": {
      "External id": 2485968,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226678.011, "dur": 1.260,
    "args": {
      "External id": 2485969,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226692.782, "dur": 1.845,
    "args": {
      "External id": 2485970,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226706.824, "dur": 0.840,
    "args": {
      "External id": 2485971,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226800.356, "dur": 2772.034,
    "args": {
      "External id": 2485972,"Record function id": 0, "Ev Idx": 2771
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.19)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226819.208, "dur": 1044.844,
    "args": {
      "External id": 2485973,"Record function id": 0, "Ev Idx": 2772
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.19)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226834.366, "dur": 362.138,
    "args": {
      "External id": 2485974,"Record function id": 0, "Ev Idx": 2773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226919.909, "dur": 6.096,
    "args": {
      "External id": 2485975,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226929.272, "dur": 1.505,
    "args": {
      "External id": 2485976,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226932.698, "dur": 1.311,
    "args": {
      "External id": 2485977,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226935.729, "dur": 1.124,
    "args": {
      "External id": 2485978,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226938.414, "dur": 1.133,
    "args": {
      "External id": 2485979,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226947.538, "dur": 0.980,
    "args": {
      "External id": 2485980,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226950.514, "dur": 2.086,
    "args": {
      "External id": 2485981,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226953.902, "dur": 1.339,
    "args": {
      "External id": 2485982,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226956.667, "dur": 2.616,
    "args": {
      "External id": 2485983,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226960.985, "dur": 1.128,
    "args": {
      "External id": 2485984,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657226979.334, "dur": 183.428,
    "args": {
      "External id": 2485985,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227027.968, "dur": 130.085,
    "args": {
      "External id": 2485986,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227046.253, "dur": 12.636,
    "args": {
      "External id": 2485987,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227062.285, "dur": 66.848,
    "args": {
      "External id": 2485988,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227065.023, "dur": 63.720,
    "args": {
      "External id": 2485989,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227069.304, "dur": 6.003,
    "args": {
      "External id": 2485990,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227077.007, "dur": 51.245,
    "args": {
      "External id": 2485991,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2790
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.18", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227288.078, "dur": 566.933,
    "args": {
      "External id": 2485992,"Record function id": 0, "Ev Idx": 2791
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.18)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227304.899, "dur": 536.934,
    "args": {
      "External id": 2485993,"Record function id": 0, "Ev Idx": 2792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227368.480, "dur": 4.607,
    "args": {
      "External id": 2485994,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227389.679, "dur": 33.174,
    "args": {
      "External id": 2485995,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227394.995, "dur": 1.523,
    "args": {
      "External id": 2485996,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227399.211, "dur": 0.680,
    "args": {
      "External id": 2485997,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227401.815, "dur": 0.381,
    "args": {
      "External id": 2485998,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227404.268, "dur": 2.046,
    "args": {
      "External id": 2485999,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227408.441, "dur": 0.533,
    "args": {
      "External id": 2486000,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227410.560, "dur": 0.435,
    "args": {
      "External id": 2486001,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227413.067, "dur": 0.352,
    "args": {
      "External id": 2486002,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227415.827, "dur": 0.367,
    "args": {
      "External id": 2486003,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227417.862, "dur": 0.352,
    "args": {
      "External id": 2486004,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227433.672, "dur": 29.338,
    "args": {
      "External id": 2486005,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227494.464, "dur": 125.832,
    "args": {
      "External id": 2486006,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227504.275, "dur": 3.418,
    "args": {
      "External id": 2486007,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227512.712, "dur": 10.828,
    "args": {
      "External id": 2486008,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227516.992, "dur": 6.116,
    "args": {
      "External id": 2486009,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227521.202, "dur": 0.784,
    "args": {
      "External id": 2486010,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227547.305, "dur": 31.141,
    "args": {
      "External id": 2486011,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227550.154, "dur": 2.798,
    "args": {
      "External id": 2486012,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227555.089, "dur": 0.701,
    "args": {
      "External id": 2486013,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227557.667, "dur": 0.373,
    "args": {
      "External id": 2486014,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227559.840, "dur": 0.771,
    "args": {
      "External id": 2486015,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227562.536, "dur": 0.434,
    "args": {
      "External id": 2486016,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227564.714, "dur": 0.675,
    "args": {
      "External id": 2486017,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227567.123, "dur": 0.403,
    "args": {
      "External id": 2486018,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227569.426, "dur": 0.360,
    "args": {
      "External id": 2486019,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227571.537, "dur": 2.449,
    "args": {
      "External id": 2486020,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227589.538, "dur": 22.977,
    "args": {
      "External id": 2486021,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227668.861, "dur": 109.182,
    "args": {
      "External id": 2486022,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227691.453, "dur": 82.952,
    "args": {
      "External id": 2486023,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2822, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227701.665, "dur": 68.868,
    "args": {
      "External id": 2486024,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227792.270, "dur": 1.913,
    "args": {
      "External id": 2486025,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2824, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657227871.664, "dur": 1676.352,
    "args": {
      "External id": 2486026,"Sequence number": 24740460, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2825
    }
  },
  {
    "ph": "f", "id": 188, "pid": 1336757, "tid": 1381170, "ts": 1514657227871.664,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228019.350, "dur": 106.496,
    "args": {
      "External id": 2486027,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228166.664, "dur": 43.898,
    "args": {
      "External id": 2486028,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228230.523, "dur": 50.041,
    "args": {
      "External id": 2486029,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228291.465, "dur": 34.219,
    "args": {
      "External id": 2486030,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228333.094, "dur": 48.179,
    "args": {
      "External id": 2486031,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228388.658, "dur": 28.267,
    "args": {
      "External id": 2486032,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228427.817, "dur": 43.285,
    "args": {
      "External id": 2486033,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228492.808, "dur": 23.576,
    "args": {
      "External id": 2486034,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228549.935, "dur": 32.506,
    "args": {
      "External id": 2486035,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228604.085, "dur": 21.051,
    "args": {
      "External id": 2486036,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228637.986, "dur": 16.866,
    "args": {
      "External id": 2486037,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228666.340, "dur": 33.029,
    "args": {
      "External id": 2486038,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228702.980, "dur": 34.566,
    "args": {
      "External id": 2486039,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228765.138, "dur": 171.388,
    "args": {
      "External id": 2486040,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228841.220, "dur": 5.877,
    "args": {
      "External id": 2486041,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228849.175, "dur": 2.474,
    "args": {
      "External id": 2486042,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657228968.577, "dur": 59.905,
    "args": {
      "External id": 2486043,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229042.496, "dur": 18.024,
    "args": {
      "External id": 2486044,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229070.975, "dur": 41.056,
    "args": {
      "External id": 2486045,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229121.058, "dur": 37.124,
    "args": {
      "External id": 2486046,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229165.691, "dur": 22.536,
    "args": {
      "External id": 2486047,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229193.290, "dur": 31.890,
    "args": {
      "External id": 2486048,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229231.645, "dur": 33.613,
    "args": {
      "External id": 2486049,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229281.594, "dur": 39.478,
    "args": {
      "External id": 2486050,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229340.216, "dur": 26.601,
    "args": {
      "External id": 2486051,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229383.594, "dur": 24.564,
    "args": {
      "External id": 2486052,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229426.056, "dur": 19.016,
    "args": {
      "External id": 2486053,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229460.831, "dur": 15.138,
    "args": {
      "External id": 2486054,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229488.677, "dur": 17.659,
    "args": {
      "External id": 2486055,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 2854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229595.389, "dur": 17.290,
    "args": {
      "External id": 2486056,"Record function id": 0, "Ev Idx": 2855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229599.206, "dur": 12.533,
    "args": {
      "External id": 2486057,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229602.830, "dur": 7.964,
    "args": {
      "External id": 2486058,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229604.521, "dur": 6.151,
    "args": {
      "External id": 2486059,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229616.652, "dur": 5.735,
    "args": {
      "External id": 2486060,"Record function id": 0, "Ev Idx": 2859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229618.413, "dur": 3.526,
    "args": {
      "External id": 2486061,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229619.123, "dur": 2.302,
    "args": {
      "External id": 2486062,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229620.017, "dur": 1.288,
    "args": {
      "External id": 2486063,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229626.030, "dur": 4.362,
    "args": {
      "External id": 2486064,"Record function id": 0, "Ev Idx": 2863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229627.415, "dur": 2.570,
    "args": {
      "External id": 2486065,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229628.085, "dur": 1.406,
    "args": {
      "External id": 2486066,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229628.436, "dur": 0.968,
    "args": {
      "External id": 2486067,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 2866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229633.646, "dur": 4.099,
    "args": {
      "External id": 2486068,"Record function id": 0, "Ev Idx": 2867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229634.892, "dur": 2.445,
    "args": {
      "External id": 2486069,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229635.785, "dur": 1.130,
    "args": {
      "External id": 2486070,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229636.162, "dur": 0.666,
    "args": {
      "External id": 2486071,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 2870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229640.821, "dur": 3.865,
    "args": {
      "External id": 2486072,"Record function id": 0, "Ev Idx": 2871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229642.152, "dur": 2.111,
    "args": {
      "External id": 2486073,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229642.611, "dur": 1.165,
    "args": {
      "External id": 2486074,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229643.118, "dur": 0.590,
    "args": {
      "External id": 2486075,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229648.033, "dur": 4.012,
    "args": {
      "External id": 2486076,"Record function id": 0, "Ev Idx": 2875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229649.377, "dur": 2.224,
    "args": {
      "External id": 2486077,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229649.824, "dur": 1.332,
    "args": {
      "External id": 2486078,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229650.277, "dur": 0.769,
    "args": {
      "External id": 2486079,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229655.514, "dur": 3.769,
    "args": {
      "External id": 2486080,"Record function id": 0, "Ev Idx": 2879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229656.785, "dur": 2.098,
    "args": {
      "External id": 2486081,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229657.407, "dur": 1.054,
    "args": {
      "External id": 2486082,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229657.703, "dur": 0.634,
    "args": {
      "External id": 2486083,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229662.525, "dur": 5.975,
    "args": {
      "External id": 2486084,"Record function id": 0, "Ev Idx": 2883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229663.988, "dur": 4.096,
    "args": {
      "External id": 2486085,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229664.443, "dur": 3.225,
    "args": {
      "External id": 2486086,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229664.742, "dur": 2.837,
    "args": {
      "External id": 2486087,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 2886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229671.916, "dur": 3.888,
    "args": {
      "External id": 2486088,"Record function id": 0, "Ev Idx": 2887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229672.981, "dur": 2.440,
    "args": {
      "External id": 2486089,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229673.435, "dur": 1.566,
    "args": {
      "External id": 2486090,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229673.890, "dur": 1.016,
    "args": {
      "External id": 2486091,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 2890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229679.647, "dur": 37665.401,
    "args": {
      "External id": 2486092,"Record function id": 0, "Sequence number": 24740459, "Fwd thread id": 1, "Ev Idx": 2891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229681.185, "dur": 37655.025,
    "args": {
      "External id": 2486093,"Sequence number": 24740459, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2892
    }
  },
  {
    "ph": "f", "id": 189, "pid": 1336757, "tid": 1381170, "ts": 1514657229681.185,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.19)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229709.544, "dur": 37.141,
    "args": {
      "External id": 2486094,"Record function id": 0, "Ev Idx": 2893
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.19)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229754.786, "dur": 68.046,
    "args": {
      "External id": 2486095,"Record function id": 0, "Ev Idx": 2894
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.19)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229828.787, "dur": 37498.764,
    "args": {
      "External id": 2486096,"Record function id": 0, "Ev Idx": 2895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229914.386, "dur": 6.423,
    "args": {
      "External id": 2486097,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229930.267, "dur": 4.643,
    "args": {
      "External id": 2486098,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229949.328, "dur": 36516.887,
    "args": {
      "External id": 2486099,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657229963.189, "dur": 36492.301,
    "args": {
      "External id": 2486100,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 2899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657230086.072, "dur": 15.157,
    "args": {
      "External id": 2486101,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657230108.063, "dur": 36304.403,
    "args": {
      "External id": 2486102,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 2901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657230110.966, "dur": 36300.546,
    "args": {
      "External id": 2486103,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 2902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657230115.076, "dur": 6.398,
    "args": {
      "External id": 2486104,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657230123.256, "dur": 36284.954,
    "args": {
      "External id": 2486105,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 2904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657266577.170, "dur": 10.266,
    "args": {
      "External id": 2486106,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 2905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657266580.516, "dur": 6.420,
    "args": {
      "External id": 2486107,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657266618.089, "dur": 361.322,
    "args": {
      "External id": 2486108,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 2907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657266645.337, "dur": 328.493,
    "args": {
      "External id": 2486109,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2908, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657266656.834, "dur": 311.190,
    "args": {
      "External id": 2486110,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 2909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267033.257, "dur": 3.293,
    "args": {
      "External id": 2486111,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2910, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267098.920, "dur": 6.688,
    "args": {
      "External id": 2486112,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267155.252, "dur": 1.711,
    "args": {
      "External id": 2486113,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267175.677, "dur": 1.108,
    "args": {
      "External id": 2486114,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267190.606, "dur": 1.013,
    "args": {
      "External id": 2486115,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267203.541, "dur": 0.892,
    "args": {
      "External id": 2486116,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267218.939, "dur": 0.982,
    "args": {
      "External id": 2486117,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267232.376, "dur": 1.032,
    "args": {
      "External id": 2486118,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267246.697, "dur": 2.163,
    "args": {
      "External id": 2486119,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267261.832, "dur": 1.015,
    "args": {
      "External id": 2486120,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267360.272, "dur": 2857.441,
    "args": {
      "External id": 2486121,"Record function id": 0, "Ev Idx": 2920
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.18)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267381.042, "dur": 1080.287,
    "args": {
      "External id": 2486122,"Record function id": 0, "Ev Idx": 2921
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.18)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267396.090, "dur": 346.758,
    "args": {
      "External id": 2486123,"Record function id": 0, "Ev Idx": 2922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267481.551, "dur": 4.366,
    "args": {
      "External id": 2486124,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 2923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267489.340, "dur": 1.427,
    "args": {
      "External id": 2486125,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267492.751, "dur": 1.349,
    "args": {
      "External id": 2486126,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267495.798, "dur": 1.005,
    "args": {
      "External id": 2486127,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267498.361, "dur": 1.404,
    "args": {
      "External id": 2486128,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267502.033, "dur": 1.065,
    "args": {
      "External id": 2486129,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 2928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267504.862, "dur": 2.105,
    "args": {
      "External id": 2486130,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 2929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267508.325, "dur": 3.055,
    "args": {
      "External id": 2486131,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267512.748, "dur": 1.009,
    "args": {
      "External id": 2486132,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267515.339, "dur": 0.911,
    "args": {
      "External id": 2486133,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 2932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267550.565, "dur": 159.751,
    "args": {
      "External id": 2486134,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267566.810, "dur": 139.187,
    "args": {
      "External id": 2486135,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 2934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267593.729, "dur": 12.433,
    "args": {
      "External id": 2486136,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267610.040, "dur": 67.392,
    "args": {
      "External id": 2486137,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 2936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267612.550, "dur": 64.474,
    "args": {
      "External id": 2486138,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 2937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267617.062, "dur": 5.442,
    "args": {
      "External id": 2486139,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267624.304, "dur": 52.028,
    "args": {
      "External id": 2486140,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 2939
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.17", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267831.454, "dur": 621.659,
    "args": {
      "External id": 2486141,"Record function id": 0, "Ev Idx": 2940
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.17)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267849.285, "dur": 590.283,
    "args": {
      "External id": 2486142,"Record function id": 0, "Ev Idx": 2941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267912.200, "dur": 5.049,
    "args": {
      "External id": 2486143,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267933.156, "dur": 40.662,
    "args": {
      "External id": 2486144,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267938.459, "dur": 1.949,
    "args": {
      "External id": 2486145,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267942.714, "dur": 0.477,
    "args": {
      "External id": 2486146,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267945.550, "dur": 2.701,
    "args": {
      "External id": 2486147,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267950.224, "dur": 0.635,
    "args": {
      "External id": 2486148,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267952.683, "dur": 0.647,
    "args": {
      "External id": 2486149,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267955.184, "dur": 0.454,
    "args": {
      "External id": 2486150,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267962.364, "dur": 0.423,
    "args": {
      "External id": 2486151,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267964.630, "dur": 0.668,
    "args": {
      "External id": 2486152,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657267967.184, "dur": 0.559,
    "args": {
      "External id": 2486153,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268026.255, "dur": 37.394,
    "args": {
      "External id": 2486154,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268101.962, "dur": 112.202,
    "args": {
      "External id": 2486155,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 2954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268113.193, "dur": 4.218,
    "args": {
      "External id": 2486156,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268122.867, "dur": 13.409,
    "args": {
      "External id": 2486157,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268127.120, "dur": 8.719,
    "args": {
      "External id": 2486158,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 2957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268131.529, "dur": 2.724,
    "args": {
      "External id": 2486159,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 2958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268143.955, "dur": 28.590,
    "args": {
      "External id": 2486160,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 2959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268146.597, "dur": 0.661,
    "args": {
      "External id": 2486161,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268149.142, "dur": 0.826,
    "args": {
      "External id": 2486162,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268151.730, "dur": 0.693,
    "args": {
      "External id": 2486163,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268154.272, "dur": 0.572,
    "args": {
      "External id": 2486164,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268156.668, "dur": 0.389,
    "args": {
      "External id": 2486165,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268158.927, "dur": 0.403,
    "args": {
      "External id": 2486166,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268160.876, "dur": 0.498,
    "args": {
      "External id": 2486167,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268163.191, "dur": 2.651,
    "args": {
      "External id": 2486168,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268167.508, "dur": 0.400,
    "args": {
      "External id": 2486169,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 2968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268187.711, "dur": 18.997,
    "args": {
      "External id": 2486170,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 2969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268261.294, "dur": 112.622,
    "args": {
      "External id": 2486171,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 2970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268286.689, "dur": 83.628,
    "args": {
      "External id": 2486172,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 2971, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268296.762, "dur": 69.511,
    "args": {
      "External id": 2486173,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 2972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268390.071, "dur": 1.788,
    "args": {
      "External id": 2486174,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 2973, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268469.546, "dur": 1724.274,
    "args": {
      "External id": 2486175,"Sequence number": 24740458, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 2974
    }
  },
  {
    "ph": "f", "id": 190, "pid": 1336757, "tid": 1381170, "ts": 1514657268469.546,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268600.289, "dur": 104.429,
    "args": {
      "External id": 2486176,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 2975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268744.249, "dur": 41.721,
    "args": {
      "External id": 2486177,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 2976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268806.553, "dur": 52.593,
    "args": {
      "External id": 2486178,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 2977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268869.613, "dur": 34.693,
    "args": {
      "External id": 2486179,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268912.421, "dur": 46.964,
    "args": {
      "External id": 2486180,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657268967.401, "dur": 68.792,
    "args": {
      "External id": 2486181,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 2980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269050.864, "dur": 48.552,
    "args": {
      "External id": 2486182,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 2981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269124.238, "dur": 25.795,
    "args": {
      "External id": 2486183,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 2982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269169.498, "dur": 30.577,
    "args": {
      "External id": 2486184,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269220.654, "dur": 21.041,
    "args": {
      "External id": 2486185,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 2984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269255.778, "dur": 16.856,
    "args": {
      "External id": 2486186,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 2985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269285.346, "dur": 31.590,
    "args": {
      "External id": 2486187,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269320.392, "dur": 35.202,
    "args": {
      "External id": 2486188,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269383.042, "dur": 190.086,
    "args": {
      "External id": 2486189,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 2988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269460.118, "dur": 5.659,
    "args": {
      "External id": 2486190,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269468.013, "dur": 2.226,
    "args": {
      "External id": 2486191,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 2990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269609.081, "dur": 25.260,
    "args": {
      "External id": 2486192,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269645.673, "dur": 17.543,
    "args": {
      "External id": 2486193,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 2992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269674.676, "dur": 41.109,
    "args": {
      "External id": 2486194,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269722.500, "dur": 38.309,
    "args": {
      "External id": 2486195,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269768.537, "dur": 39.038,
    "args": {
      "External id": 2486196,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269818.706, "dur": 36.619,
    "args": {
      "External id": 2486197,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269864.691, "dur": 21.288,
    "args": {
      "External id": 2486198,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 2997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269895.815, "dur": 31.846,
    "args": {
      "External id": 2486199,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 2998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657269945.886, "dur": 26.526,
    "args": {
      "External id": 2486200,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 2999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270035.687, "dur": 28.162,
    "args": {
      "External id": 2486201,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270080.852, "dur": 19.255,
    "args": {
      "External id": 2486202,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270116.009, "dur": 15.715,
    "args": {
      "External id": 2486203,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270144.484, "dur": 18.539,
    "args": {
      "External id": 2486204,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270240.488, "dur": 16.023,
    "args": {
      "External id": 2486205,"Record function id": 0, "Ev Idx": 3004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270244.454, "dur": 11.191,
    "args": {
      "External id": 2486206,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270248.584, "dur": 6.021,
    "args": {
      "External id": 2486207,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270250.251, "dur": 4.238,
    "args": {
      "External id": 2486208,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270261.179, "dur": 4.656,
    "args": {
      "External id": 2486209,"Record function id": 0, "Ev Idx": 3008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270262.719, "dur": 2.667,
    "args": {
      "External id": 2486210,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270263.421, "dur": 1.410,
    "args": {
      "External id": 2486211,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270263.960, "dur": 0.782,
    "args": {
      "External id": 2486212,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270269.690, "dur": 4.505,
    "args": {
      "External id": 2486213,"Record function id": 0, "Ev Idx": 3012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270271.214, "dur": 2.566,
    "args": {
      "External id": 2486214,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270271.926, "dur": 1.451,
    "args": {
      "External id": 2486215,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270272.629, "dur": 0.678,
    "args": {
      "External id": 2486216,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270277.640, "dur": 4.146,
    "args": {
      "External id": 2486217,"Record function id": 0, "Ev Idx": 3016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270278.852, "dur": 2.518,
    "args": {
      "External id": 2486218,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270279.504, "dur": 1.461,
    "args": {
      "External id": 2486219,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270280.020, "dur": 0.859,
    "args": {
      "External id": 2486220,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270285.057, "dur": 3.772,
    "args": {
      "External id": 2486221,"Record function id": 0, "Ev Idx": 3020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270286.267, "dur": 2.115,
    "args": {
      "External id": 2486222,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270286.796, "dur": 1.182,
    "args": {
      "External id": 2486223,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270287.213, "dur": 0.680,
    "args": {
      "External id": 2486224,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270292.151, "dur": 3.719,
    "args": {
      "External id": 2486225,"Record function id": 0, "Ev Idx": 3024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270293.420, "dur": 2.037,
    "args": {
      "External id": 2486226,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270293.930, "dur": 1.025,
    "args": {
      "External id": 2486227,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270294.189, "dur": 0.670,
    "args": {
      "External id": 2486228,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270299.113, "dur": 6.675,
    "args": {
      "External id": 2486229,"Record function id": 0, "Ev Idx": 3028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270300.353, "dur": 4.996,
    "args": {
      "External id": 2486230,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270301.106, "dur": 3.751,
    "args": {
      "External id": 2486231,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270301.573, "dur": 3.162,
    "args": {
      "External id": 2486232,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270309.172, "dur": 4.313,
    "args": {
      "External id": 2486233,"Record function id": 0, "Ev Idx": 3032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270310.380, "dur": 2.689,
    "args": {
      "External id": 2486234,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270311.059, "dur": 1.586,
    "args": {
      "External id": 2486235,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270311.336, "dur": 1.217,
    "args": {
      "External id": 2486236,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270316.989, "dur": 3.782,
    "args": {
      "External id": 2486237,"Record function id": 0, "Ev Idx": 3036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270318.417, "dur": 1.953,
    "args": {
      "External id": 2486238,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270318.909, "dur": 0.950,
    "args": {
      "External id": 2486239,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270319.222, "dur": 0.567,
    "args": {
      "External id": 2486240,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270325.072, "dur": 37099.494,
    "args": {
      "External id": 2486241,"Record function id": 0, "Sequence number": 24740457, "Fwd thread id": 1, "Ev Idx": 3040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270326.449, "dur": 37089.419,
    "args": {
      "External id": 2486242,"Sequence number": 24740457, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3041
    }
  },
  {
    "ph": "f", "id": 191, "pid": 1336757, "tid": 1381170, "ts": 1514657270326.449,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.18)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270354.040, "dur": 36.750,
    "args": {
      "External id": 2486243,"Record function id": 0, "Ev Idx": 3042
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.18)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270398.367, "dur": 67.461,
    "args": {
      "External id": 2486244,"Record function id": 0, "Ev Idx": 3043
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.18)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270472.233, "dur": 36936.120,
    "args": {
      "External id": 2486245,"Record function id": 0, "Ev Idx": 3044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270574.648, "dur": 7.748,
    "args": {
      "External id": 2486246,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270592.771, "dur": 4.831,
    "args": {
      "External id": 2486247,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270612.833, "dur": 35906.890,
    "args": {
      "External id": 2486248,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270662.078, "dur": 35848.280,
    "args": {
      "External id": 2486249,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270711.044, "dur": 14.243,
    "args": {
      "External id": 2486250,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270731.715, "dur": 35738.504,
    "args": {
      "External id": 2486251,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270734.534, "dur": 35734.995,
    "args": {
      "External id": 2486252,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270738.780, "dur": 5.434,
    "args": {
      "External id": 2486253,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657270746.101, "dur": 35719.828,
    "args": {
      "External id": 2486254,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657306622.295, "dur": 9.080,
    "args": {
      "External id": 2486255,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657306625.365, "dur": 5.493,
    "args": {
      "External id": 2486256,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657306660.389, "dur": 430.401,
    "args": {
      "External id": 2486257,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657306691.457, "dur": 393.842,
    "args": {
      "External id": 2486258,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3057, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657306703.102, "dur": 375.359,
    "args": {
      "External id": 2486259,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307114.894, "dur": 2.813,
    "args": {
      "External id": 2486260,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3059, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307181.410, "dur": 8.676,
    "args": {
      "External id": 2486261,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307236.952, "dur": 1.424,
    "args": {
      "External id": 2486262,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307256.410, "dur": 1.134,
    "args": {
      "External id": 2486263,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307271.588, "dur": 1.621,
    "args": {
      "External id": 2486264,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307286.909, "dur": 3.055,
    "args": {
      "External id": 2486265,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307301.863, "dur": 1.253,
    "args": {
      "External id": 2486266,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307316.620, "dur": 1.168,
    "args": {
      "External id": 2486267,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307331.020, "dur": 1.964,
    "args": {
      "External id": 2486268,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307344.738, "dur": 2.898,
    "args": {
      "External id": 2486269,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307439.238, "dur": 2861.707,
    "args": {
      "External id": 2486270,"Record function id": 0, "Ev Idx": 3069
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.17)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307458.733, "dur": 1126.754,
    "args": {
      "External id": 2486271,"Record function id": 0, "Ev Idx": 3070
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.17)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307474.093, "dur": 377.649,
    "args": {
      "External id": 2486272,"Record function id": 0, "Ev Idx": 3071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307609.809, "dur": 4.924,
    "args": {
      "External id": 2486273,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307618.715, "dur": 1.556,
    "args": {
      "External id": 2486274,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307621.884, "dur": 1.234,
    "args": {
      "External id": 2486275,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307624.729, "dur": 0.754,
    "args": {
      "External id": 2486276,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307627.140, "dur": 1.186,
    "args": {
      "External id": 2486277,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307631.606, "dur": 0.832,
    "args": {
      "External id": 2486278,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307634.084, "dur": 4.520,
    "args": {
      "External id": 2486279,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307640.017, "dur": 1.168,
    "args": {
      "External id": 2486280,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307642.845, "dur": 0.809,
    "args": {
      "External id": 2486281,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307645.334, "dur": 0.880,
    "args": {
      "External id": 2486282,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307665.309, "dur": 151.502,
    "args": {
      "External id": 2486283,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307682.314, "dur": 129.921,
    "args": {
      "External id": 2486284,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307698.486, "dur": 12.951,
    "args": {
      "External id": 2486285,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307714.985, "dur": 67.275,
    "args": {
      "External id": 2486286,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307717.658, "dur": 64.291,
    "args": {
      "External id": 2486287,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307721.677, "dur": 6.144,
    "args": {
      "External id": 2486288,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307729.777, "dur": 51.598,
    "args": {
      "External id": 2486289,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3088
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.16", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307949.320, "dur": 627.278,
    "args": {
      "External id": 2486290,"Record function id": 0, "Ev Idx": 3089
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.16)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657307966.893, "dur": 595.139,
    "args": {
      "External id": 2486291,"Record function id": 0, "Ev Idx": 3090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308072.583, "dur": 6.496,
    "args": {
      "External id": 2486292,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308095.099, "dur": 34.222,
    "args": {
      "External id": 2486293,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308100.724, "dur": 1.695,
    "args": {
      "External id": 2486294,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308104.788, "dur": 2.540,
    "args": {
      "External id": 2486295,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308108.999, "dur": 0.584,
    "args": {
      "External id": 2486296,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308111.567, "dur": 0.460,
    "args": {
      "External id": 2486297,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308113.812, "dur": 0.392,
    "args": {
      "External id": 2486298,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308116.017, "dur": 0.462,
    "args": {
      "External id": 2486299,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308118.183, "dur": 0.484,
    "args": {
      "External id": 2486300,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308120.658, "dur": 0.359,
    "args": {
      "External id": 2486301,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308123.176, "dur": 0.464,
    "args": {
      "External id": 2486302,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308140.302, "dur": 32.552,
    "args": {
      "External id": 2486303,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308207.447, "dur": 109.123,
    "args": {
      "External id": 2486304,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308217.988, "dur": 5.347,
    "args": {
      "External id": 2486305,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308228.311, "dur": 10.936,
    "args": {
      "External id": 2486306,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308232.779, "dur": 6.063,
    "args": {
      "External id": 2486307,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308236.669, "dur": 0.814,
    "args": {
      "External id": 2486308,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308245.914, "dur": 26.734,
    "args": {
      "External id": 2486309,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308248.155, "dur": 0.548,
    "args": {
      "External id": 2486310,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308250.352, "dur": 0.535,
    "args": {
      "External id": 2486311,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308252.600, "dur": 0.480,
    "args": {
      "External id": 2486312,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308254.954, "dur": 0.434,
    "args": {
      "External id": 2486313,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308257.044, "dur": 0.454,
    "args": {
      "External id": 2486314,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308259.247, "dur": 0.515,
    "args": {
      "External id": 2486315,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308261.392, "dur": 2.168,
    "args": {
      "External id": 2486316,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308265.283, "dur": 0.442,
    "args": {
      "External id": 2486317,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308267.168, "dur": 0.449,
    "args": {
      "External id": 2486318,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308289.629, "dur": 19.520,
    "args": {
      "External id": 2486319,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308365.272, "dur": 111.864,
    "args": {
      "External id": 2486320,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308389.744, "dur": 84.185,
    "args": {
      "External id": 2486321,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3120, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308400.051, "dur": 69.690,
    "args": {
      "External id": 2486322,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308493.584, "dur": 1.666,
    "args": {
      "External id": 2486323,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3122, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308594.396, "dur": 1684.953,
    "args": {
      "External id": 2486324,"Sequence number": 24740456, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3123
    }
  },
  {
    "ph": "f", "id": 192, "pid": 1336757, "tid": 1381170, "ts": 1514657308594.396,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308705.124, "dur": 104.537,
    "args": {
      "External id": 2486325,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308845.665, "dur": 38.872,
    "args": {
      "External id": 2486326,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308905.233, "dur": 49.972,
    "args": {
      "External id": 2486327,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657308965.351, "dur": 72.678,
    "args": {
      "External id": 2486328,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309049.120, "dur": 50.998,
    "args": {
      "External id": 2486329,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309111.004, "dur": 29.645,
    "args": {
      "External id": 2486330,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309149.286, "dur": 42.855,
    "args": {
      "External id": 2486331,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309215.942, "dur": 23.290,
    "args": {
      "External id": 2486332,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309256.984, "dur": 29.477,
    "args": {
      "External id": 2486333,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309306.613, "dur": 20.749,
    "args": {
      "External id": 2486334,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309339.765, "dur": 16.889,
    "args": {
      "External id": 2486335,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309368.162, "dur": 29.536,
    "args": {
      "External id": 2486336,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309401.371, "dur": 34.493,
    "args": {
      "External id": 2486337,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309462.617, "dur": 196.872,
    "args": {
      "External id": 2486338,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309555.077, "dur": 6.493,
    "args": {
      "External id": 2486339,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309563.920, "dur": 4.547,
    "args": {
      "External id": 2486340,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309694.088, "dur": 24.551,
    "args": {
      "External id": 2486341,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309744.089, "dur": 18.564,
    "args": {
      "External id": 2486342,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309772.342, "dur": 39.517,
    "args": {
      "External id": 2486343,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309819.532, "dur": 36.936,
    "args": {
      "External id": 2486344,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309864.058, "dur": 32.662,
    "args": {
      "External id": 2486345,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309909.487, "dur": 41.144,
    "args": {
      "External id": 2486346,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657309958.172, "dur": 21.707,
    "args": {
      "External id": 2486347,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310021.893, "dur": 41.066,
    "args": {
      "External id": 2486348,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310083.110, "dur": 28.069,
    "args": {
      "External id": 2486349,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310128.490, "dur": 24.025,
    "args": {
      "External id": 2486350,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310168.853, "dur": 19.553,
    "args": {
      "External id": 2486351,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310203.539, "dur": 15.878,
    "args": {
      "External id": 2486352,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310233.118, "dur": 17.947,
    "args": {
      "External id": 2486353,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310324.639, "dur": 15.179,
    "args": {
      "External id": 2486354,"Record function id": 0, "Ev Idx": 3153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310327.888, "dur": 11.141,
    "args": {
      "External id": 2486355,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310331.925, "dur": 6.051,
    "args": {
      "External id": 2486356,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310333.863, "dur": 3.971,
    "args": {
      "External id": 2486357,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310343.827, "dur": 4.924,
    "args": {
      "External id": 2486358,"Record function id": 0, "Ev Idx": 3157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310345.383, "dur": 2.928,
    "args": {
      "External id": 2486359,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310346.251, "dur": 1.567,
    "args": {
      "External id": 2486360,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310346.797, "dur": 0.929,
    "args": {
      "External id": 2486361,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310352.074, "dur": 4.865,
    "args": {
      "External id": 2486362,"Record function id": 0, "Ev Idx": 3161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310353.619, "dur": 2.901,
    "args": {
      "External id": 2486363,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310354.327, "dur": 1.724,
    "args": {
      "External id": 2486364,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310355.024, "dur": 0.959,
    "args": {
      "External id": 2486365,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310360.149, "dur": 6.302,
    "args": {
      "External id": 2486366,"Record function id": 0, "Ev Idx": 3165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310361.504, "dur": 4.546,
    "args": {
      "External id": 2486367,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310362.077, "dur": 3.526,
    "args": {
      "External id": 2486368,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310362.579, "dur": 2.960,
    "args": {
      "External id": 2486369,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310369.650, "dur": 4.240,
    "args": {
      "External id": 2486370,"Record function id": 0, "Ev Idx": 3169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310371.210, "dur": 2.248,
    "args": {
      "External id": 2486371,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310371.705, "dur": 1.303,
    "args": {
      "External id": 2486372,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310371.969, "dur": 0.974,
    "args": {
      "External id": 2486373,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310377.000, "dur": 4.300,
    "args": {
      "External id": 2486374,"Record function id": 0, "Ev Idx": 3173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310378.328, "dur": 2.548,
    "args": {
      "External id": 2486375,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310378.890, "dur": 1.445,
    "args": {
      "External id": 2486376,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310379.550, "dur": 0.688,
    "args": {
      "External id": 2486377,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310384.460, "dur": 4.108,
    "args": {
      "External id": 2486378,"Record function id": 0, "Ev Idx": 3177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310385.620, "dur": 2.495,
    "args": {
      "External id": 2486379,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310386.258, "dur": 1.371,
    "args": {
      "External id": 2486380,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310386.553, "dur": 0.991,
    "args": {
      "External id": 2486381,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310391.712, "dur": 4.204,
    "args": {
      "External id": 2486382,"Record function id": 0, "Ev Idx": 3181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310392.887, "dur": 2.600,
    "args": {
      "External id": 2486383,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310393.547, "dur": 1.515,
    "args": {
      "External id": 2486384,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310394.215, "dur": 0.713,
    "args": {
      "External id": 2486385,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310399.055, "dur": 4.189,
    "args": {
      "External id": 2486386,"Record function id": 0, "Ev Idx": 3185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310400.084, "dur": 2.718,
    "args": {
      "External id": 2486387,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310400.818, "dur": 1.553,
    "args": {
      "External id": 2486388,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310401.533, "dur": 0.722,
    "args": {
      "External id": 2486389,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310407.108, "dur": 36845.276,
    "args": {
      "External id": 2486390,"Record function id": 0, "Sequence number": 24740455, "Fwd thread id": 1, "Ev Idx": 3189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310408.627, "dur": 36835.199,
    "args": {
      "External id": 2486391,"Sequence number": 24740455, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3190
    }
  },
  {
    "ph": "f", "id": 193, "pid": 1336757, "tid": 1381170, "ts": 1514657310408.627,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.17)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310437.737, "dur": 35.245,
    "args": {
      "External id": 2486392,"Record function id": 0, "Ev Idx": 3191
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.17)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310480.958, "dur": 84.626,
    "args": {
      "External id": 2486393,"Record function id": 0, "Ev Idx": 3192
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.17)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310573.553, "dur": 36662.760,
    "args": {
      "External id": 2486394,"Record function id": 0, "Ev Idx": 3193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310660.806, "dur": 7.293,
    "args": {
      "External id": 2486395,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310678.455, "dur": 5.020,
    "args": {
      "External id": 2486396,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310697.661, "dur": 35716.984,
    "args": {
      "External id": 2486397,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310711.148, "dur": 35694.911,
    "args": {
      "External id": 2486398,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310763.671, "dur": 15.246,
    "args": {
      "External id": 2486399,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310785.170, "dur": 35579.487,
    "args": {
      "External id": 2486400,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310787.956, "dur": 35576.001,
    "args": {
      "External id": 2486401,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310791.956, "dur": 4.816,
    "args": {
      "External id": 2486402,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657310798.593, "dur": 35561.628,
    "args": {
      "External id": 2486403,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657346506.012, "dur": 9.952,
    "args": {
      "External id": 2486404,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657346509.743, "dur": 5.833,
    "args": {
      "External id": 2486405,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657346559.725, "dur": 335.558,
    "args": {
      "External id": 2486406,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657346586.561, "dur": 304.350,
    "args": {
      "External id": 2486407,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3206, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657346598.817, "dur": 286.892,
    "args": {
      "External id": 2486408,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657346914.720, "dur": 2.383,
    "args": {
      "External id": 2486409,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3208, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657346972.939, "dur": 6.483,
    "args": {
      "External id": 2486410,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347061.623, "dur": 2.384,
    "args": {
      "External id": 2486411,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347083.387, "dur": 1.048,
    "args": {
      "External id": 2486412,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347099.711, "dur": 1.019,
    "args": {
      "External id": 2486413,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347112.940, "dur": 1.084,
    "args": {
      "External id": 2486414,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347125.758, "dur": 1.097,
    "args": {
      "External id": 2486415,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347140.120, "dur": 1.510,
    "args": {
      "External id": 2486416,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347158.127, "dur": 2.118,
    "args": {
      "External id": 2486417,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347173.480, "dur": 1.138,
    "args": {
      "External id": 2486418,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347269.027, "dur": 2811.879,
    "args": {
      "External id": 2486419,"Record function id": 0, "Ev Idx": 3218
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.16)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347287.486, "dur": 1045.284,
    "args": {
      "External id": 2486420,"Record function id": 0, "Ev Idx": 3219
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.16)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347301.226, "dur": 347.221,
    "args": {
      "External id": 2486421,"Record function id": 0, "Ev Idx": 3220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347385.760, "dur": 4.188,
    "args": {
      "External id": 2486422,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347393.087, "dur": 1.332,
    "args": {
      "External id": 2486423,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347396.164, "dur": 1.349,
    "args": {
      "External id": 2486424,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347399.576, "dur": 3.551,
    "args": {
      "External id": 2486425,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347404.700, "dur": 1.102,
    "args": {
      "External id": 2486426,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347407.703, "dur": 1.328,
    "args": {
      "External id": 2486427,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347410.766, "dur": 2.028,
    "args": {
      "External id": 2486428,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347414.474, "dur": 1.327,
    "args": {
      "External id": 2486429,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347417.367, "dur": 1.068,
    "args": {
      "External id": 2486430,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347419.864, "dur": 0.846,
    "args": {
      "External id": 2486431,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347438.154, "dur": 175.235,
    "args": {
      "External id": 2486432,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347454.130, "dur": 153.661,
    "args": {
      "External id": 2486433,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347477.576, "dur": 12.274,
    "args": {
      "External id": 2486434,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347493.335, "dur": 84.063,
    "args": {
      "External id": 2486435,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347496.058, "dur": 80.965,
    "args": {
      "External id": 2486436,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347500.354, "dur": 7.469,
    "args": {
      "External id": 2486437,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347509.594, "dur": 66.526,
    "args": {
      "External id": 2486438,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3237
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.15", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347743.796, "dur": 580.131,
    "args": {
      "External id": 2486439,"Record function id": 0, "Ev Idx": 3238
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.15)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347759.876, "dur": 551.734,
    "args": {
      "External id": 2486440,"Record function id": 0, "Ev Idx": 3239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347820.279, "dur": 5.363,
    "args": {
      "External id": 2486441,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347841.100, "dur": 33.718,
    "args": {
      "External id": 2486442,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347846.320, "dur": 1.610,
    "args": {
      "External id": 2486443,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347850.551, "dur": 0.603,
    "args": {
      "External id": 2486444,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347853.156, "dur": 0.665,
    "args": {
      "External id": 2486445,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347855.653, "dur": 0.605,
    "args": {
      "External id": 2486446,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347858.127, "dur": 0.597,
    "args": {
      "External id": 2486447,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347860.380, "dur": 0.464,
    "args": {
      "External id": 2486448,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347862.527, "dur": 2.480,
    "args": {
      "External id": 2486449,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347867.169, "dur": 0.341,
    "args": {
      "External id": 2486450,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347869.403, "dur": 0.540,
    "args": {
      "External id": 2486451,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347885.105, "dur": 29.934,
    "args": {
      "External id": 2486452,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347947.018, "dur": 146.390,
    "args": {
      "External id": 2486453,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347957.432, "dur": 3.022,
    "args": {
      "External id": 2486454,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347965.321, "dur": 9.950,
    "args": {
      "External id": 2486455,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347969.520, "dur": 5.346,
    "args": {
      "External id": 2486456,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657347973.254, "dur": 0.475,
    "args": {
      "External id": 2486457,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348020.475, "dur": 27.640,
    "args": {
      "External id": 2486458,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348023.707, "dur": 0.781,
    "args": {
      "External id": 2486459,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348026.642, "dur": 0.615,
    "args": {
      "External id": 2486460,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348029.007, "dur": 0.533,
    "args": {
      "External id": 2486461,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348031.385, "dur": 2.364,
    "args": {
      "External id": 2486462,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348035.229, "dur": 0.316,
    "args": {
      "External id": 2486463,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348037.098, "dur": 0.545,
    "args": {
      "External id": 2486464,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348039.103, "dur": 0.332,
    "args": {
      "External id": 2486465,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348041.068, "dur": 0.569,
    "args": {
      "External id": 2486466,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348043.280, "dur": 0.433,
    "args": {
      "External id": 2486467,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348062.559, "dur": 22.567,
    "args": {
      "External id": 2486468,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348139.770, "dur": 109.628,
    "args": {
      "External id": 2486469,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348161.790, "dur": 84.168,
    "args": {
      "External id": 2486470,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3269, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348171.535, "dur": 70.225,
    "args": {
      "External id": 2486471,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348263.043, "dur": 1.828,
    "args": {
      "External id": 2486472,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3271, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348340.765, "dur": 1716.381,
    "args": {
      "External id": 2486473,"Sequence number": 24740454, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3272
    }
  },
  {
    "ph": "f", "id": 194, "pid": 1336757, "tid": 1381170, "ts": 1514657348340.765,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348449.221, "dur": 120.868,
    "args": {
      "External id": 2486474,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348611.740, "dur": 41.438,
    "args": {
      "External id": 2486475,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348672.728, "dur": 56.074,
    "args": {
      "External id": 2486476,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348738.829, "dur": 34.882,
    "args": {
      "External id": 2486477,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348783.843, "dur": 47.947,
    "args": {
      "External id": 2486478,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348857.772, "dur": 28.795,
    "args": {
      "External id": 2486479,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348895.009, "dur": 44.636,
    "args": {
      "External id": 2486480,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657348962.642, "dur": 64.940,
    "args": {
      "External id": 2486481,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349052.948, "dur": 32.547,
    "args": {
      "External id": 2486482,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349107.723, "dur": 20.858,
    "args": {
      "External id": 2486483,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349141.574, "dur": 16.447,
    "args": {
      "External id": 2486484,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349167.375, "dur": 34.450,
    "args": {
      "External id": 2486485,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349205.231, "dur": 33.592,
    "args": {
      "External id": 2486486,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349268.687, "dur": 174.526,
    "args": {
      "External id": 2486487,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349348.149, "dur": 5.766,
    "args": {
      "External id": 2486488,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349355.884, "dur": 2.235,
    "args": {
      "External id": 2486489,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349475.431, "dur": 24.070,
    "args": {
      "External id": 2486490,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349511.706, "dur": 31.197,
    "args": {
      "External id": 2486491,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349555.276, "dur": 40.768,
    "args": {
      "External id": 2486492,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349604.238, "dur": 36.006,
    "args": {
      "External id": 2486493,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349650.403, "dur": 34.943,
    "args": {
      "External id": 2486494,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349696.455, "dur": 41.184,
    "args": {
      "External id": 2486495,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349745.031, "dur": 21.153,
    "args": {
      "External id": 2486496,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349772.945, "dur": 34.765,
    "args": {
      "External id": 2486497,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349827.461, "dur": 23.563,
    "args": {
      "External id": 2486498,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349870.031, "dur": 24.346,
    "args": {
      "External id": 2486499,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349909.083, "dur": 18.186,
    "args": {
      "External id": 2486500,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349946.448, "dur": 14.218,
    "args": {
      "External id": 2486501,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657349973.673, "dur": 52.906,
    "args": {
      "External id": 2486502,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350103.513, "dur": 16.186,
    "args": {
      "External id": 2486503,"Record function id": 0, "Ev Idx": 3302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350107.600, "dur": 11.209,
    "args": {
      "External id": 2486504,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350111.453, "dur": 6.195,
    "args": {
      "External id": 2486505,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350113.341, "dur": 4.222,
    "args": {
      "External id": 2486506,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350123.810, "dur": 5.609,
    "args": {
      "External id": 2486507,"Record function id": 0, "Ev Idx": 3306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350125.812, "dur": 3.175,
    "args": {
      "External id": 2486508,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350126.539, "dur": 1.994,
    "args": {
      "External id": 2486509,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350127.102, "dur": 1.342,
    "args": {
      "External id": 2486510,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350132.852, "dur": 7.479,
    "args": {
      "External id": 2486511,"Record function id": 0, "Ev Idx": 3310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350134.193, "dur": 5.715,
    "args": {
      "External id": 2486512,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350135.409, "dur": 4.055,
    "args": {
      "External id": 2486513,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350136.125, "dur": 3.281,
    "args": {
      "External id": 2486514,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350143.484, "dur": 5.241,
    "args": {
      "External id": 2486515,"Record function id": 0, "Ev Idx": 3314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350145.218, "dur": 3.092,
    "args": {
      "External id": 2486516,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350145.832, "dur": 2.065,
    "args": {
      "External id": 2486517,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350146.791, "dur": 1.034,
    "args": {
      "External id": 2486518,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350152.117, "dur": 4.513,
    "args": {
      "External id": 2486519,"Record function id": 0, "Ev Idx": 3318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350153.830, "dur": 2.390,
    "args": {
      "External id": 2486520,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350154.436, "dur": 1.402,
    "args": {
      "External id": 2486521,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350155.007, "dur": 0.732,
    "args": {
      "External id": 2486522,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350160.070, "dur": 4.670,
    "args": {
      "External id": 2486523,"Record function id": 0, "Ev Idx": 3322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350161.778, "dur": 2.550,
    "args": {
      "External id": 2486524,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350162.384, "dur": 1.527,
    "args": {
      "External id": 2486525,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350163.031, "dur": 0.788,
    "args": {
      "External id": 2486526,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350168.135, "dur": 4.123,
    "args": {
      "External id": 2486527,"Record function id": 0, "Ev Idx": 3326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350169.595, "dur": 2.239,
    "args": {
      "External id": 2486528,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350170.275, "dur": 1.166,
    "args": {
      "External id": 2486529,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350170.701, "dur": 0.647,
    "args": {
      "External id": 2486530,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350175.759, "dur": 4.627,
    "args": {
      "External id": 2486531,"Record function id": 0, "Ev Idx": 3330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350177.177, "dur": 2.781,
    "args": {
      "External id": 2486532,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350177.694, "dur": 1.877,
    "args": {
      "External id": 2486533,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350178.400, "dur": 1.058,
    "args": {
      "External id": 2486534,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350183.678, "dur": 3.566,
    "args": {
      "External id": 2486535,"Record function id": 0, "Ev Idx": 3334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350184.718, "dur": 2.121,
    "args": {
      "External id": 2486536,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350185.513, "dur": 0.923,
    "args": {
      "External id": 2486537,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350185.810, "dur": 0.564,
    "args": {
      "External id": 2486538,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350191.276, "dur": 37615.933,
    "args": {
      "External id": 2486539,"Record function id": 0, "Sequence number": 24740453, "Fwd thread id": 1, "Ev Idx": 3338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350192.616, "dur": 37606.193,
    "args": {
      "External id": 2486540,"Sequence number": 24740453, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3339
    }
  },
  {
    "ph": "f", "id": 195, "pid": 1336757, "tid": 1381170, "ts": 1514657350192.616,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.16)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350221.308, "dur": 36.626,
    "args": {
      "External id": 2486541,"Record function id": 0, "Ev Idx": 3340
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.16)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350265.452, "dur": 62.999,
    "args": {
      "External id": 2486542,"Record function id": 0, "Ev Idx": 3341
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.16)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350334.473, "dur": 37456.428,
    "args": {
      "External id": 2486543,"Record function id": 0, "Ev Idx": 3342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350422.002, "dur": 6.962,
    "args": {
      "External id": 2486544,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350438.176, "dur": 5.953,
    "args": {
      "External id": 2486545,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350458.326, "dur": 36452.397,
    "args": {
      "External id": 2486546,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350471.556, "dur": 36430.072,
    "args": {
      "External id": 2486547,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350522.596, "dur": 31.205,
    "args": {
      "External id": 2486548,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350561.334, "dur": 36300.209,
    "args": {
      "External id": 2486549,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350564.202, "dur": 36296.760,
    "args": {
      "External id": 2486550,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350568.344, "dur": 6.117,
    "args": {
      "External id": 2486551,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657350576.368, "dur": 36281.199,
    "args": {
      "External id": 2486552,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387030.181, "dur": 9.485,
    "args": {
      "External id": 2486553,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387033.215, "dur": 5.893,
    "args": {
      "External id": 2486554,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387070.467, "dur": 382.064,
    "args": {
      "External id": 2486555,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387101.465, "dur": 346.471,
    "args": {
      "External id": 2486556,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3355, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387113.410, "dur": 328.987,
    "args": {
      "External id": 2486557,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387483.271, "dur": 2.554,
    "args": {
      "External id": 2486558,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3357, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387565.781, "dur": 6.719,
    "args": {
      "External id": 2486559,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387618.805, "dur": 1.374,
    "args": {
      "External id": 2486560,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387637.580, "dur": 2.996,
    "args": {
      "External id": 2486561,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387655.135, "dur": 0.882,
    "args": {
      "External id": 2486562,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387669.676, "dur": 0.808,
    "args": {
      "External id": 2486563,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387683.008, "dur": 1.281,
    "args": {
      "External id": 2486564,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387697.783, "dur": 2.793,
    "args": {
      "External id": 2486565,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387713.308, "dur": 2.241,
    "args": {
      "External id": 2486566,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387727.241, "dur": 0.880,
    "args": {
      "External id": 2486567,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387823.454, "dur": 2809.197,
    "args": {
      "External id": 2486568,"Record function id": 0, "Ev Idx": 3367
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.15)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387842.678, "dur": 1045.463,
    "args": {
      "External id": 2486569,"Record function id": 0, "Ev Idx": 3368
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.15)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387856.360, "dur": 371.160,
    "args": {
      "External id": 2486570,"Record function id": 0, "Ev Idx": 3369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387942.792, "dur": 3.854,
    "args": {
      "External id": 2486571,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387949.802, "dur": 1.350,
    "args": {
      "External id": 2486572,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387952.886, "dur": 3.570,
    "args": {
      "External id": 2486573,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387958.222, "dur": 0.981,
    "args": {
      "External id": 2486574,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387960.902, "dur": 1.028,
    "args": {
      "External id": 2486575,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387963.818, "dur": 1.167,
    "args": {
      "External id": 2486576,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387966.976, "dur": 1.774,
    "args": {
      "External id": 2486577,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387970.398, "dur": 0.978,
    "args": {
      "External id": 2486578,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387972.902, "dur": 1.040,
    "args": {
      "External id": 2486579,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657387975.566, "dur": 1.018,
    "args": {
      "External id": 2486580,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388031.606, "dur": 161.123,
    "args": {
      "External id": 2486581,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388048.910, "dur": 139.200,
    "args": {
      "External id": 2486582,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388071.554, "dur": 14.308,
    "args": {
      "External id": 2486583,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388089.472, "dur": 69.516,
    "args": {
      "External id": 2486584,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388092.256, "dur": 66.360,
    "args": {
      "External id": 2486585,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388096.244, "dur": 5.851,
    "args": {
      "External id": 2486586,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388104.049, "dur": 54.022,
    "args": {
      "External id": 2486587,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3386
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.14", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388323.574, "dur": 556.586,
    "args": {
      "External id": 2486588,"Record function id": 0, "Ev Idx": 3387
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.14)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388341.142, "dur": 526.687,
    "args": {
      "External id": 2486589,"Record function id": 0, "Ev Idx": 3388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388403.688, "dur": 4.798,
    "args": {
      "External id": 2486590,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388423.929, "dur": 33.170,
    "args": {
      "External id": 2486591,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388429.099, "dur": 1.670,
    "args": {
      "External id": 2486592,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388433.405, "dur": 0.508,
    "args": {
      "External id": 2486593,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388436.087, "dur": 0.470,
    "args": {
      "External id": 2486594,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388438.722, "dur": 0.447,
    "args": {
      "External id": 2486595,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388440.837, "dur": 0.460,
    "args": {
      "External id": 2486596,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388443.130, "dur": 2.362,
    "args": {
      "External id": 2486597,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388447.492, "dur": 0.425,
    "args": {
      "External id": 2486598,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388449.682, "dur": 0.423,
    "args": {
      "External id": 2486599,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388451.896, "dur": 0.540,
    "args": {
      "External id": 2486600,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388467.404, "dur": 28.584,
    "args": {
      "External id": 2486601,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388527.055, "dur": 123.657,
    "args": {
      "External id": 2486602,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388554.413, "dur": 4.940,
    "args": {
      "External id": 2486603,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388565.098, "dur": 11.264,
    "args": {
      "External id": 2486604,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388569.173, "dur": 6.700,
    "args": {
      "External id": 2486605,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388573.350, "dur": 0.967,
    "args": {
      "External id": 2486606,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388584.012, "dur": 26.411,
    "args": {
      "External id": 2486607,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388586.604, "dur": 0.612,
    "args": {
      "External id": 2486608,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388589.261, "dur": 0.371,
    "args": {
      "External id": 2486609,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388591.293, "dur": 1.739,
    "args": {
      "External id": 2486610,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388594.861, "dur": 0.390,
    "args": {
      "External id": 2486611,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388597.065, "dur": 0.404,
    "args": {
      "External id": 2486612,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388599.102, "dur": 0.396,
    "args": {
      "External id": 2486613,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388601.258, "dur": 0.426,
    "args": {
      "External id": 2486614,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388603.271, "dur": 0.334,
    "args": {
      "External id": 2486615,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388605.718, "dur": 0.353,
    "args": {
      "External id": 2486616,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388621.482, "dur": 21.374,
    "args": {
      "External id": 2486617,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388697.501, "dur": 106.716,
    "args": {
      "External id": 2486618,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388718.698, "dur": 82.170,
    "args": {
      "External id": 2486619,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3418, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388728.632, "dur": 68.181,
    "args": {
      "External id": 2486620,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388817.866, "dur": 2.077,
    "args": {
      "External id": 2486621,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3420, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657388895.454, "dur": 1715.123,
    "args": {
      "External id": 2486622,"Sequence number": 24740452, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3421
    }
  },
  {
    "ph": "f", "id": 196, "pid": 1336757, "tid": 1381170, "ts": 1514657388895.454,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389047.243, "dur": 105.467,
    "args": {
      "External id": 2486623,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389190.674, "dur": 41.106,
    "args": {
      "External id": 2486624,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389248.938, "dur": 51.439,
    "args": {
      "External id": 2486625,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389312.236, "dur": 33.743,
    "args": {
      "External id": 2486626,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389353.744, "dur": 47.426,
    "args": {
      "External id": 2486627,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389408.600, "dur": 27.785,
    "args": {
      "External id": 2486628,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389443.704, "dur": 41.446,
    "args": {
      "External id": 2486629,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389510.660, "dur": 39.949,
    "args": {
      "External id": 2486630,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389573.146, "dur": 31.203,
    "args": {
      "External id": 2486631,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389626.273, "dur": 21.174,
    "args": {
      "External id": 2486632,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389662.805, "dur": 16.652,
    "args": {
      "External id": 2486633,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389688.615, "dur": 34.489,
    "args": {
      "External id": 2486634,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389726.765, "dur": 34.986,
    "args": {
      "External id": 2486635,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389806.999, "dur": 171.186,
    "args": {
      "External id": 2486636,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389882.680, "dur": 5.975,
    "args": {
      "External id": 2486637,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657389890.984, "dur": 2.340,
    "args": {
      "External id": 2486638,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390054.567, "dur": 28.025,
    "args": {
      "External id": 2486639,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390094.378, "dur": 17.319,
    "args": {
      "External id": 2486640,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390121.845, "dur": 43.590,
    "args": {
      "External id": 2486641,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390172.930, "dur": 50.199,
    "args": {
      "External id": 2486642,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390238.975, "dur": 28.380,
    "args": {
      "External id": 2486643,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390273.266, "dur": 36.325,
    "args": {
      "External id": 2486644,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390316.466, "dur": 22.040,
    "args": {
      "External id": 2486645,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390350.526, "dur": 30.898,
    "args": {
      "External id": 2486646,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390404.854, "dur": 24.014,
    "args": {
      "External id": 2486647,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390444.923, "dur": 23.050,
    "args": {
      "External id": 2486648,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390481.350, "dur": 18.860,
    "args": {
      "External id": 2486649,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390514.306, "dur": 32.777,
    "args": {
      "External id": 2486650,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390562.622, "dur": 18.795,
    "args": {
      "External id": 2486651,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390655.812, "dur": 15.980,
    "args": {
      "External id": 2486652,"Record function id": 0, "Ev Idx": 3451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390659.692, "dur": 11.148,
    "args": {
      "External id": 2486653,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390663.884, "dur": 6.089,
    "args": {
      "External id": 2486654,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390665.835, "dur": 4.014,
    "args": {
      "External id": 2486655,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390675.559, "dur": 7.850,
    "args": {
      "External id": 2486656,"Record function id": 0, "Ev Idx": 3455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390677.334, "dur": 5.625,
    "args": {
      "External id": 2486657,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390678.095, "dur": 4.218,
    "args": {
      "External id": 2486658,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390679.243, "dur": 2.941,
    "args": {
      "External id": 2486659,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390686.816, "dur": 4.980,
    "args": {
      "External id": 2486660,"Record function id": 0, "Ev Idx": 3459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390688.514, "dur": 2.870,
    "args": {
      "External id": 2486661,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390689.272, "dur": 1.677,
    "args": {
      "External id": 2486662,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390689.869, "dur": 1.008,
    "args": {
      "External id": 2486663,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390695.135, "dur": 3.939,
    "args": {
      "External id": 2486664,"Record function id": 0, "Ev Idx": 3463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390696.436, "dur": 2.223,
    "args": {
      "External id": 2486665,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390697.062, "dur": 1.206,
    "args": {
      "External id": 2486666,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390697.387, "dur": 0.805,
    "args": {
      "External id": 2486667,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390702.175, "dur": 4.088,
    "args": {
      "External id": 2486668,"Record function id": 0, "Ev Idx": 3467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390703.640, "dur": 2.213,
    "args": {
      "External id": 2486669,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390704.273, "dur": 1.188,
    "args": {
      "External id": 2486670,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390704.751, "dur": 0.638,
    "args": {
      "External id": 2486671,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390709.679, "dur": 4.256,
    "args": {
      "External id": 2486672,"Record function id": 0, "Ev Idx": 3471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390711.176, "dur": 2.312,
    "args": {
      "External id": 2486673,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390711.654, "dur": 1.412,
    "args": {
      "External id": 2486674,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390712.348, "dur": 0.600,
    "args": {
      "External id": 2486675,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390717.158, "dur": 3.926,
    "args": {
      "External id": 2486676,"Record function id": 0, "Ev Idx": 3475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390718.478, "dur": 2.184,
    "args": {
      "External id": 2486677,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390718.954, "dur": 1.306,
    "args": {
      "External id": 2486678,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390719.446, "dur": 0.741,
    "args": {
      "External id": 2486679,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390724.180, "dur": 4.120,
    "args": {
      "External id": 2486680,"Record function id": 0, "Ev Idx": 3479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390725.499, "dur": 2.354,
    "args": {
      "External id": 2486681,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390725.975, "dur": 1.454,
    "args": {
      "External id": 2486682,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390726.430, "dur": 0.894,
    "args": {
      "External id": 2486683,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390731.352, "dur": 5.783,
    "args": {
      "External id": 2486684,"Record function id": 0, "Ev Idx": 3483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390732.603, "dur": 4.078,
    "args": {
      "External id": 2486685,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390733.235, "dur": 3.040,
    "args": {
      "External id": 2486686,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390733.815, "dur": 2.339,
    "args": {
      "External id": 2486687,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390741.473, "dur": 35917.978,
    "args": {
      "External id": 2486688,"Record function id": 0, "Sequence number": 24740451, "Fwd thread id": 1, "Ev Idx": 3487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390742.744, "dur": 35907.801,
    "args": {
      "External id": 2486689,"Sequence number": 24740451, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3488
    }
  },
  {
    "ph": "f", "id": 197, "pid": 1336757, "tid": 1381170, "ts": 1514657390742.744,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.15)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390771.297, "dur": 36.233,
    "args": {
      "External id": 2486690,"Record function id": 0, "Ev Idx": 3489
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.15)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390814.901, "dur": 61.238,
    "args": {
      "External id": 2486691,"Record function id": 0, "Ev Idx": 3490
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.15)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390881.485, "dur": 35761.511,
    "args": {
      "External id": 2486692,"Record function id": 0, "Ev Idx": 3491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657390966.077, "dur": 6.672,
    "args": {
      "External id": 2486693,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657391031.841, "dur": 5.060,
    "args": {
      "External id": 2486694,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657391054.109, "dur": 34758.770,
    "args": {
      "External id": 2486695,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657391068.876, "dur": 34735.613,
    "args": {
      "External id": 2486696,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657391122.618, "dur": 14.203,
    "args": {
      "External id": 2486697,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657391143.138, "dur": 34620.353,
    "args": {
      "External id": 2486698,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657391146.047, "dur": 34616.838,
    "args": {
      "External id": 2486699,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657391150.303, "dur": 6.378,
    "args": {
      "External id": 2486700,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657391158.599, "dur": 34600.650,
    "args": {
      "External id": 2486701,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657425904.468, "dur": 9.331,
    "args": {
      "External id": 2486702,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657425907.796, "dur": 5.575,
    "args": {
      "External id": 2486703,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657425944.399, "dur": 373.599,
    "args": {
      "External id": 2486704,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657425970.078, "dur": 342.909,
    "args": {
      "External id": 2486705,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3504, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657425980.887, "dur": 325.997,
    "args": {
      "External id": 2486706,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426338.749, "dur": 2.664,
    "args": {
      "External id": 2486707,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3506, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426400.128, "dur": 6.606,
    "args": {
      "External id": 2486708,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426453.075, "dur": 3.719,
    "args": {
      "External id": 2486709,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426474.346, "dur": 1.148,
    "args": {
      "External id": 2486710,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426488.555, "dur": 0.973,
    "args": {
      "External id": 2486711,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426502.005, "dur": 1.108,
    "args": {
      "External id": 2486712,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426515.060, "dur": 2.797,
    "args": {
      "External id": 2486713,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426546.865, "dur": 1.599,
    "args": {
      "External id": 2486714,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426564.489, "dur": 3.065,
    "args": {
      "External id": 2486715,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426578.876, "dur": 0.899,
    "args": {
      "External id": 2486716,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426675.896, "dur": 2824.545,
    "args": {
      "External id": 2486717,"Record function id": 0, "Ev Idx": 3516
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.14)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426694.486, "dur": 1055.774,
    "args": {
      "External id": 2486718,"Record function id": 0, "Ev Idx": 3517
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.14)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426707.936, "dur": 363.301,
    "args": {
      "External id": 2486719,"Record function id": 0, "Ev Idx": 3518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426792.851, "dur": 5.909,
    "args": {
      "External id": 2486720,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426801.810, "dur": 1.069,
    "args": {
      "External id": 2486721,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426805.063, "dur": 1.147,
    "args": {
      "External id": 2486722,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426807.682, "dur": 1.270,
    "args": {
      "External id": 2486723,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426810.374, "dur": 1.207,
    "args": {
      "External id": 2486724,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426813.145, "dur": 1.058,
    "args": {
      "External id": 2486725,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426815.926, "dur": 2.057,
    "args": {
      "External id": 2486726,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426819.509, "dur": 0.789,
    "args": {
      "External id": 2486727,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426821.843, "dur": 3.214,
    "args": {
      "External id": 2486728,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426826.590, "dur": 0.951,
    "args": {
      "External id": 2486729,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426845.786, "dur": 189.701,
    "args": {
      "External id": 2486730,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426861.681, "dur": 168.319,
    "args": {
      "External id": 2486731,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426883.653, "dur": 11.784,
    "args": {
      "External id": 2486732,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426898.661, "dur": 66.210,
    "args": {
      "External id": 2486733,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426901.421, "dur": 63.140,
    "args": {
      "External id": 2486734,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426905.341, "dur": 5.684,
    "args": {
      "External id": 2486735,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657426913.011, "dur": 50.948,
    "args": {
      "External id": 2486736,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3535
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.13", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427165.929, "dur": 575.535,
    "args": {
      "External id": 2486737,"Record function id": 0, "Ev Idx": 3536
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.13)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427185.109, "dur": 543.865,
    "args": {
      "External id": 2486738,"Record function id": 0, "Ev Idx": 3537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427248.067, "dur": 6.008,
    "args": {
      "External id": 2486739,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427269.449, "dur": 33.301,
    "args": {
      "External id": 2486740,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427274.699, "dur": 1.800,
    "args": {
      "External id": 2486741,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427279.057, "dur": 0.616,
    "args": {
      "External id": 2486742,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427281.356, "dur": 0.448,
    "args": {
      "External id": 2486743,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427283.432, "dur": 2.366,
    "args": {
      "External id": 2486744,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427287.483, "dur": 0.421,
    "args": {
      "External id": 2486745,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427289.722, "dur": 0.470,
    "args": {
      "External id": 2486746,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427291.913, "dur": 0.552,
    "args": {
      "External id": 2486747,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427294.354, "dur": 0.825,
    "args": {
      "External id": 2486748,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427297.290, "dur": 0.582,
    "args": {
      "External id": 2486749,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427313.936, "dur": 32.138,
    "args": {
      "External id": 2486750,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427378.599, "dur": 109.991,
    "args": {
      "External id": 2486751,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427389.124, "dur": 3.156,
    "args": {
      "External id": 2486752,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427402.974, "dur": 10.740,
    "args": {
      "External id": 2486753,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427407.525, "dur": 5.731,
    "args": {
      "External id": 2486754,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427411.381, "dur": 0.670,
    "args": {
      "External id": 2486755,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427423.564, "dur": 26.810,
    "args": {
      "External id": 2486756,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427425.840, "dur": 2.265,
    "args": {
      "External id": 2486757,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427429.665, "dur": 0.849,
    "args": {
      "External id": 2486758,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427432.256, "dur": 0.410,
    "args": {
      "External id": 2486759,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427434.353, "dur": 0.306,
    "args": {
      "External id": 2486760,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427436.303, "dur": 0.329,
    "args": {
      "External id": 2486761,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427438.028, "dur": 0.332,
    "args": {
      "External id": 2486762,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427440.194, "dur": 0.405,
    "args": {
      "External id": 2486763,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427442.388, "dur": 0.564,
    "args": {
      "External id": 2486764,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427444.392, "dur": 2.443,
    "args": {
      "External id": 2486765,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427462.548, "dur": 18.898,
    "args": {
      "External id": 2486766,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427550.550, "dur": 111.417,
    "args": {
      "External id": 2486767,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427572.299, "dur": 86.166,
    "args": {
      "External id": 2486768,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3567, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427582.899, "dur": 71.481,
    "args": {
      "External id": 2486769,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427676.326, "dur": 1.786,
    "args": {
      "External id": 2486770,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3569, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427759.035, "dur": 1719.457,
    "args": {
      "External id": 2486771,"Sequence number": 24740450, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3570
    }
  },
  {
    "ph": "f", "id": 198, "pid": 1336757, "tid": 1381170, "ts": 1514657427759.035,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657427869.272, "dur": 101.487,
    "args": {
      "External id": 2486772,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428048.234, "dur": 43.306,
    "args": {
      "External id": 2486773,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428113.742, "dur": 57.287,
    "args": {
      "External id": 2486774,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428181.189, "dur": 34.406,
    "args": {
      "External id": 2486775,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428223.715, "dur": 48.206,
    "args": {
      "External id": 2486776,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428280.401, "dur": 28.707,
    "args": {
      "External id": 2486777,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428318.974, "dur": 43.847,
    "args": {
      "External id": 2486778,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428386.720, "dur": 23.077,
    "args": {
      "External id": 2486779,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428427.410, "dur": 29.872,
    "args": {
      "External id": 2486780,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428475.956, "dur": 21.271,
    "args": {
      "External id": 2486781,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428510.246, "dur": 16.098,
    "args": {
      "External id": 2486782,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428553.697, "dur": 34.826,
    "args": {
      "External id": 2486783,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428592.066, "dur": 35.084,
    "args": {
      "External id": 2486784,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428682.479, "dur": 177.261,
    "args": {
      "External id": 2486785,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428762.593, "dur": 5.901,
    "args": {
      "External id": 2486786,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428770.821, "dur": 2.218,
    "args": {
      "External id": 2486787,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428894.537, "dur": 24.664,
    "args": {
      "External id": 2486788,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428930.302, "dur": 16.578,
    "args": {
      "External id": 2486789,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657428958.484, "dur": 76.552,
    "args": {
      "External id": 2486790,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429047.431, "dur": 41.429,
    "args": {
      "External id": 2486791,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429097.415, "dur": 36.668,
    "args": {
      "External id": 2486792,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429145.451, "dur": 36.409,
    "args": {
      "External id": 2486793,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429189.739, "dur": 21.877,
    "args": {
      "External id": 2486794,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429221.648, "dur": 33.942,
    "args": {
      "External id": 2486795,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429274.445, "dur": 27.612,
    "args": {
      "External id": 2486796,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429326.953, "dur": 25.678,
    "args": {
      "External id": 2486797,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429369.644, "dur": 18.603,
    "args": {
      "External id": 2486798,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429404.846, "dur": 15.539,
    "args": {
      "External id": 2486799,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429432.608, "dur": 17.210,
    "args": {
      "External id": 2486800,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429523.584, "dur": 34.580,
    "args": {
      "External id": 2486801,"Record function id": 0, "Ev Idx": 3600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429526.473, "dur": 30.228,
    "args": {
      "External id": 2486802,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429546.906, "dur": 8.481,
    "args": {
      "External id": 2486803,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429548.995, "dur": 6.066,
    "args": {
      "External id": 2486804,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429564.137, "dur": 5.600,
    "args": {
      "External id": 2486805,"Record function id": 0, "Ev Idx": 3604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429565.668, "dur": 3.666,
    "args": {
      "External id": 2486806,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429566.720, "dur": 1.980,
    "args": {
      "External id": 2486807,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429567.405, "dur": 1.204,
    "args": {
      "External id": 2486808,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429573.017, "dur": 4.561,
    "args": {
      "External id": 2486809,"Record function id": 0, "Ev Idx": 3608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429574.330, "dur": 2.836,
    "args": {
      "External id": 2486810,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429575.014, "dur": 1.510,
    "args": {
      "External id": 2486811,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429575.527, "dur": 0.908,
    "args": {
      "External id": 2486812,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429580.870, "dur": 4.065,
    "args": {
      "External id": 2486813,"Record function id": 0, "Ev Idx": 3612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429582.254, "dur": 2.273,
    "args": {
      "External id": 2486814,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429582.788, "dur": 1.219,
    "args": {
      "External id": 2486815,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429583.301, "dur": 0.591,
    "args": {
      "External id": 2486816,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429588.036, "dur": 4.219,
    "args": {
      "External id": 2486817,"Record function id": 0, "Ev Idx": 3616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429589.460, "dur": 2.373,
    "args": {
      "External id": 2486818,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429589.962, "dur": 1.408,
    "args": {
      "External id": 2486819,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429590.486, "dur": 0.781,
    "args": {
      "External id": 2486820,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429595.354, "dur": 3.915,
    "args": {
      "External id": 2486821,"Record function id": 0, "Ev Idx": 3620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429596.535, "dur": 2.338,
    "args": {
      "External id": 2486822,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429597.060, "dur": 1.282,
    "args": {
      "External id": 2486823,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429597.672, "dur": 0.556,
    "args": {
      "External id": 2486824,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429602.553, "dur": 3.553,
    "args": {
      "External id": 2486825,"Record function id": 0, "Ev Idx": 3624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429603.644, "dur": 2.046,
    "args": {
      "External id": 2486826,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429604.233, "dur": 1.023,
    "args": {
      "External id": 2486827,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429604.520, "dur": 0.630,
    "args": {
      "External id": 2486828,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429609.207, "dur": 5.725,
    "args": {
      "External id": 2486829,"Record function id": 0, "Ev Idx": 3628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429610.499, "dur": 4.020,
    "args": {
      "External id": 2486830,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429611.086, "dur": 2.893,
    "args": {
      "External id": 2486831,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429611.352, "dur": 2.558,
    "args": {
      "External id": 2486832,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429617.974, "dur": 4.069,
    "args": {
      "External id": 2486833,"Record function id": 0, "Ev Idx": 3632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429619.076, "dur": 2.552,
    "args": {
      "External id": 2486834,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429619.538, "dur": 1.352,
    "args": {
      "External id": 2486835,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429619.982, "dur": 0.822,
    "args": {
      "External id": 2486836,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429626.005, "dur": 35963.127,
    "args": {
      "External id": 2486837,"Record function id": 0, "Sequence number": 24740449, "Fwd thread id": 1, "Ev Idx": 3636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429627.328, "dur": 35952.331,
    "args": {
      "External id": 2486838,"Sequence number": 24740449, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3637
    }
  },
  {
    "ph": "f", "id": 199, "pid": 1336757, "tid": 1381170, "ts": 1514657429627.328,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.14)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429658.429, "dur": 36.862,
    "args": {
      "External id": 2486839,"Record function id": 0, "Ev Idx": 3638
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.14)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429702.725, "dur": 60.332,
    "args": {
      "External id": 2486840,"Record function id": 0, "Ev Idx": 3639
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.14)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429768.456, "dur": 35802.434,
    "args": {
      "External id": 2486841,"Record function id": 0, "Ev Idx": 3640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429852.519, "dur": 6.915,
    "args": {
      "External id": 2486842,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429869.022, "dur": 4.659,
    "args": {
      "External id": 2486843,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429887.917, "dur": 34882.167,
    "args": {
      "External id": 2486844,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429901.518, "dur": 34860.206,
    "args": {
      "External id": 2486845,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429950.023, "dur": 13.092,
    "args": {
      "External id": 2486846,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429969.495, "dur": 34752.521,
    "args": {
      "External id": 2486847,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429972.421, "dur": 34748.571,
    "args": {
      "External id": 2486848,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657429976.533, "dur": 55.609,
    "args": {
      "External id": 2486849,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657430035.094, "dur": 34682.031,
    "args": {
      "External id": 2486850,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657464858.342, "dur": 9.212,
    "args": {
      "External id": 2486851,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657464861.596, "dur": 5.646,
    "args": {
      "External id": 2486852,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657464895.174, "dur": 361.259,
    "args": {
      "External id": 2486853,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657464921.004, "dur": 330.528,
    "args": {
      "External id": 2486854,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3653, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657464932.078, "dur": 313.333,
    "args": {
      "External id": 2486855,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465277.061, "dur": 2.156,
    "args": {
      "External id": 2486856,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3655, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465336.870, "dur": 6.719,
    "args": {
      "External id": 2486857,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465391.470, "dur": 1.265,
    "args": {
      "External id": 2486858,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465409.401, "dur": 0.963,
    "args": {
      "External id": 2486859,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465424.390, "dur": 1.092,
    "args": {
      "External id": 2486860,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465437.441, "dur": 0.937,
    "args": {
      "External id": 2486861,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465452.510, "dur": 0.947,
    "args": {
      "External id": 2486862,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465465.076, "dur": 0.752,
    "args": {
      "External id": 2486863,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465478.165, "dur": 1.885,
    "args": {
      "External id": 2486864,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465492.695, "dur": 1.070,
    "args": {
      "External id": 2486865,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465606.018, "dur": 2779.795,
    "args": {
      "External id": 2486866,"Record function id": 0, "Ev Idx": 3665
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.13)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465625.490, "dur": 1032.030,
    "args": {
      "External id": 2486867,"Record function id": 0, "Ev Idx": 3666
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.13)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465638.787, "dur": 317.553,
    "args": {
      "External id": 2486868,"Record function id": 0, "Ev Idx": 3667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465723.040, "dur": 4.631,
    "args": {
      "External id": 2486869,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465730.730, "dur": 1.382,
    "args": {
      "External id": 2486870,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465733.844, "dur": 1.296,
    "args": {
      "External id": 2486871,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465736.553, "dur": 0.878,
    "args": {
      "External id": 2486872,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465739.124, "dur": 0.982,
    "args": {
      "External id": 2486873,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465741.790, "dur": 0.905,
    "args": {
      "External id": 2486874,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465744.164, "dur": 1.867,
    "args": {
      "External id": 2486875,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465747.821, "dur": 2.515,
    "args": {
      "External id": 2486876,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465752.186, "dur": 0.966,
    "args": {
      "External id": 2486877,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465754.588, "dur": 0.769,
    "args": {
      "External id": 2486878,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465773.639, "dur": 152.396,
    "args": {
      "External id": 2486879,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465789.763, "dur": 132.055,
    "args": {
      "External id": 2486880,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465812.103, "dur": 11.969,
    "args": {
      "External id": 2486881,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465827.232, "dur": 65.700,
    "args": {
      "External id": 2486882,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465830.057, "dur": 62.452,
    "args": {
      "External id": 2486883,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465833.879, "dur": 5.237,
    "args": {
      "External id": 2486884,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657465840.972, "dur": 50.827,
    "args": {
      "External id": 2486885,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3684
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.12", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466087.947, "dur": 561.101,
    "args": {
      "External id": 2486886,"Record function id": 0, "Ev Idx": 3685
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.12)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466104.796, "dur": 531.906,
    "args": {
      "External id": 2486887,"Record function id": 0, "Ev Idx": 3686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466170.634, "dur": 5.738,
    "args": {
      "External id": 2486888,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466191.605, "dur": 30.856,
    "args": {
      "External id": 2486889,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466196.625, "dur": 1.716,
    "args": {
      "External id": 2486890,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466200.770, "dur": 0.407,
    "args": {
      "External id": 2486891,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466202.755, "dur": 2.730,
    "args": {
      "External id": 2486892,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466207.405, "dur": 0.458,
    "args": {
      "External id": 2486893,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466209.276, "dur": 0.337,
    "args": {
      "External id": 2486894,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466211.512, "dur": 0.352,
    "args": {
      "External id": 2486895,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466213.606, "dur": 0.386,
    "args": {
      "External id": 2486896,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466215.635, "dur": 0.435,
    "args": {
      "External id": 2486897,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466217.810, "dur": 0.375,
    "args": {
      "External id": 2486898,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466232.350, "dur": 33.479,
    "args": {
      "External id": 2486899,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466299.311, "dur": 100.092,
    "args": {
      "External id": 2486900,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466309.513, "dur": 3.983,
    "args": {
      "External id": 2486901,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466318.533, "dur": 11.821,
    "args": {
      "External id": 2486902,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466322.904, "dur": 7.003,
    "args": {
      "External id": 2486903,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466326.475, "dur": 2.169,
    "args": {
      "External id": 2486904,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466336.844, "dur": 26.023,
    "args": {
      "External id": 2486905,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466339.330, "dur": 0.418,
    "args": {
      "External id": 2486906,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466341.417, "dur": 0.583,
    "args": {
      "External id": 2486907,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466343.662, "dur": 0.454,
    "args": {
      "External id": 2486908,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466346.055, "dur": 0.446,
    "args": {
      "External id": 2486909,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466348.420, "dur": 0.382,
    "args": {
      "External id": 2486910,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466350.456, "dur": 0.754,
    "args": {
      "External id": 2486911,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466352.878, "dur": 0.332,
    "args": {
      "External id": 2486912,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466355.110, "dur": 2.309,
    "args": {
      "External id": 2486913,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466359.000, "dur": 0.554,
    "args": {
      "External id": 2486914,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466373.082, "dur": 19.218,
    "args": {
      "External id": 2486915,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466443.521, "dur": 126.368,
    "args": {
      "External id": 2486916,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466464.792, "dur": 101.367,
    "args": {
      "External id": 2486917,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3716, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466474.336, "dur": 87.024,
    "args": {
      "External id": 2486918,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466584.327, "dur": 2.138,
    "args": {
      "External id": 2486919,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3718, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466664.832, "dur": 1699.756,
    "args": {
      "External id": 2486920,"Sequence number": 24740448, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3719
    }
  },
  {
    "ph": "f", "id": 200, "pid": 1336757, "tid": 1381170, "ts": 1514657466664.832,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466773.349, "dur": 105.143,
    "args": {
      "External id": 2486921,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466913.794, "dur": 38.793,
    "args": {
      "External id": 2486922,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657466971.430, "dur": 96.343,
    "args": {
      "External id": 2486923,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467082.248, "dur": 37.373,
    "args": {
      "External id": 2486924,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467127.912, "dur": 47.319,
    "args": {
      "External id": 2486925,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467183.586, "dur": 29.110,
    "args": {
      "External id": 2486926,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467223.946, "dur": 44.983,
    "args": {
      "External id": 2486927,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467293.104, "dur": 25.991,
    "args": {
      "External id": 2486928,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467336.821, "dur": 30.194,
    "args": {
      "External id": 2486929,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467386.356, "dur": 20.015,
    "args": {
      "External id": 2486930,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467419.380, "dur": 15.583,
    "args": {
      "External id": 2486931,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467446.439, "dur": 29.235,
    "args": {
      "External id": 2486932,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467479.215, "dur": 33.669,
    "args": {
      "External id": 2486933,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467556.919, "dur": 173.578,
    "args": {
      "External id": 2486934,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467633.642, "dur": 6.608,
    "args": {
      "External id": 2486935,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467642.385, "dur": 2.328,
    "args": {
      "External id": 2486936,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467765.539, "dur": 23.968,
    "args": {
      "External id": 2486937,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467823.286, "dur": 17.162,
    "args": {
      "External id": 2486938,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467851.503, "dur": 40.418,
    "args": {
      "External id": 2486939,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467898.281, "dur": 37.082,
    "args": {
      "External id": 2486940,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467942.710, "dur": 21.747,
    "args": {
      "External id": 2486941,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657467970.211, "dur": 76.434,
    "args": {
      "External id": 2486942,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468069.600, "dur": 34.098,
    "args": {
      "External id": 2486943,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468112.675, "dur": 33.464,
    "args": {
      "External id": 2486944,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468169.251, "dur": 24.099,
    "args": {
      "External id": 2486945,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468210.252, "dur": 28.457,
    "args": {
      "External id": 2486946,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468253.834, "dur": 17.366,
    "args": {
      "External id": 2486947,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468288.740, "dur": 16.434,
    "args": {
      "External id": 2486948,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468317.493, "dur": 17.032,
    "args": {
      "External id": 2486949,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468408.730, "dur": 14.591,
    "args": {
      "External id": 2486950,"Record function id": 0, "Ev Idx": 3749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468411.919, "dur": 10.570,
    "args": {
      "External id": 2486951,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468415.851, "dur": 5.742,
    "args": {
      "External id": 2486952,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468417.628, "dur": 3.832,
    "args": {
      "External id": 2486953,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468427.302, "dur": 5.658,
    "args": {
      "External id": 2486954,"Record function id": 0, "Ev Idx": 3753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468428.771, "dur": 3.746,
    "args": {
      "External id": 2486955,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468429.574, "dur": 2.426,
    "args": {
      "External id": 2486956,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468430.807, "dur": 1.041,
    "args": {
      "External id": 2486957,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468436.188, "dur": 5.123,
    "args": {
      "External id": 2486958,"Record function id": 0, "Ev Idx": 3757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468437.628, "dur": 3.237,
    "args": {
      "External id": 2486959,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468438.187, "dur": 2.206,
    "args": {
      "External id": 2486960,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468439.080, "dur": 1.248,
    "args": {
      "External id": 2486961,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468444.453, "dur": 4.258,
    "args": {
      "External id": 2486962,"Record function id": 0, "Ev Idx": 3761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468446.002, "dur": 2.285,
    "args": {
      "External id": 2486963,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468446.552, "dur": 1.337,
    "args": {
      "External id": 2486964,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468447.082, "dur": 0.741,
    "args": {
      "External id": 2486965,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468451.872, "dur": 3.359,
    "args": {
      "External id": 2486966,"Record function id": 0, "Ev Idx": 3765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468452.803, "dur": 2.005,
    "args": {
      "External id": 2486967,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468453.264, "dur": 1.128,
    "args": {
      "External id": 2486968,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468453.748, "dur": 0.578,
    "args": {
      "External id": 2486969,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468458.313, "dur": 3.769,
    "args": {
      "External id": 2486970,"Record function id": 0, "Ev Idx": 3769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468459.365, "dur": 2.310,
    "args": {
      "External id": 2486971,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468460.097, "dur": 1.154,
    "args": {
      "External id": 2486972,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468460.554, "dur": 0.593,
    "args": {
      "External id": 2486973,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468465.246, "dur": 5.385,
    "args": {
      "External id": 2486974,"Record function id": 0, "Ev Idx": 3773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468466.161, "dur": 4.066,
    "args": {
      "External id": 2486975,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468467.075, "dur": 2.734,
    "args": {
      "External id": 2486976,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468467.387, "dur": 2.334,
    "args": {
      "External id": 2486977,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468473.778, "dur": 3.880,
    "args": {
      "External id": 2486978,"Record function id": 0, "Ev Idx": 3777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468474.862, "dur": 2.400,
    "args": {
      "External id": 2486979,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468475.328, "dur": 1.508,
    "args": {
      "External id": 2486980,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468475.884, "dur": 0.871,
    "args": {
      "External id": 2486981,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468480.691, "dur": 3.869,
    "args": {
      "External id": 2486982,"Record function id": 0, "Ev Idx": 3781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468482.032, "dur": 2.121,
    "args": {
      "External id": 2486983,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468482.500, "dur": 1.235,
    "args": {
      "External id": 2486984,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468483.039, "dur": 0.612,
    "args": {
      "External id": 2486985,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468488.262, "dur": 38397.210,
    "args": {
      "External id": 2486986,"Record function id": 0, "Sequence number": 24740447, "Fwd thread id": 1, "Ev Idx": 3785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468489.637, "dur": 38387.825,
    "args": {
      "External id": 2486987,"Sequence number": 24740447, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3786
    }
  },
  {
    "ph": "f", "id": 201, "pid": 1336757, "tid": 1381170, "ts": 1514657468489.637,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.13)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468518.566, "dur": 53.836,
    "args": {
      "External id": 2486988,"Record function id": 0, "Ev Idx": 3787
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.13)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468581.374, "dur": 66.449,
    "args": {
      "External id": 2486989,"Record function id": 0, "Ev Idx": 3788
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.13)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468653.928, "dur": 38216.071,
    "args": {
      "External id": 2486990,"Record function id": 0, "Ev Idx": 3789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468741.615, "dur": 7.081,
    "args": {
      "External id": 2486991,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468758.742, "dur": 4.923,
    "args": {
      "External id": 2486992,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468778.164, "dur": 37252.121,
    "args": {
      "External id": 2486993,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468791.835, "dur": 37230.172,
    "args": {
      "External id": 2486994,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468840.424, "dur": 13.222,
    "args": {
      "External id": 2486995,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468859.808, "dur": 37096.776,
    "args": {
      "External id": 2486996,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468862.432, "dur": 37093.410,
    "args": {
      "External id": 2486997,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468866.726, "dur": 5.004,
    "args": {
      "External id": 2486998,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657468873.697, "dur": 37078.540,
    "args": {
      "External id": 2486999,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506121.590, "dur": 9.202,
    "args": {
      "External id": 2487000,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506124.935, "dur": 5.572,
    "args": {
      "External id": 2487001,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506161.377, "dur": 403.016,
    "args": {
      "External id": 2487002,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506187.735, "dur": 370.825,
    "args": {
      "External id": 2487003,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3802, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506198.781, "dur": 353.690,
    "args": {
      "External id": 2487004,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506586.769, "dur": 2.362,
    "args": {
      "External id": 2487005,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3804, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506651.101, "dur": 8.530,
    "args": {
      "External id": 2487006,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506706.162, "dur": 1.397,
    "args": {
      "External id": 2487007,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506724.742, "dur": 0.993,
    "args": {
      "External id": 2487008,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506741.003, "dur": 0.948,
    "args": {
      "External id": 2487009,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506752.867, "dur": 2.732,
    "args": {
      "External id": 2487010,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506767.578, "dur": 1.212,
    "args": {
      "External id": 2487011,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506780.593, "dur": 1.233,
    "args": {
      "External id": 2487012,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506794.427, "dur": 2.058,
    "args": {
      "External id": 2487013,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506808.190, "dur": 2.721,
    "args": {
      "External id": 2487014,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506899.483, "dur": 2768.128,
    "args": {
      "External id": 2487015,"Record function id": 0, "Ev Idx": 3814
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.12)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506918.696, "dur": 1036.519,
    "args": {
      "External id": 2487016,"Record function id": 0, "Ev Idx": 3815
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.12)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657506931.912, "dur": 360.377,
    "args": {
      "External id": 2487017,"Record function id": 0, "Ev Idx": 3816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507051.353, "dur": 4.854,
    "args": {
      "External id": 2487018,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507059.750, "dur": 1.035,
    "args": {
      "External id": 2487019,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507062.625, "dur": 1.000,
    "args": {
      "External id": 2487020,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507065.556, "dur": 0.993,
    "args": {
      "External id": 2487021,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507068.041, "dur": 1.004,
    "args": {
      "External id": 2487022,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507070.812, "dur": 0.922,
    "args": {
      "External id": 2487023,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507073.554, "dur": 4.454,
    "args": {
      "External id": 2487024,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507079.444, "dur": 0.923,
    "args": {
      "External id": 2487025,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507082.204, "dur": 1.037,
    "args": {
      "External id": 2487026,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507085.131, "dur": 0.817,
    "args": {
      "External id": 2487027,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507104.363, "dur": 154.862,
    "args": {
      "External id": 2487028,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507120.839, "dur": 133.897,
    "args": {
      "External id": 2487029,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507142.878, "dur": 12.935,
    "args": {
      "External id": 2487030,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507158.823, "dur": 67.035,
    "args": {
      "External id": 2487031,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507161.374, "dur": 64.085,
    "args": {
      "External id": 2487032,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507165.310, "dur": 5.538,
    "args": {
      "External id": 2487033,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507172.607, "dur": 52.346,
    "args": {
      "External id": 2487034,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3833
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.11", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507384.602, "dur": 562.424,
    "args": {
      "External id": 2487035,"Record function id": 0, "Ev Idx": 3834
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.11)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507403.337, "dur": 532.123,
    "args": {
      "External id": 2487036,"Record function id": 0, "Ev Idx": 3835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507467.806, "dur": 4.817,
    "args": {
      "External id": 2487037,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507487.613, "dur": 30.658,
    "args": {
      "External id": 2487038,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507493.003, "dur": 1.802,
    "args": {
      "External id": 2487039,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507497.123, "dur": 2.481,
    "args": {
      "External id": 2487040,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507501.327, "dur": 0.579,
    "args": {
      "External id": 2487041,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507503.449, "dur": 0.416,
    "args": {
      "External id": 2487042,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507505.606, "dur": 0.418,
    "args": {
      "External id": 2487043,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507507.597, "dur": 0.589,
    "args": {
      "External id": 2487044,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507509.879, "dur": 0.392,
    "args": {
      "External id": 2487045,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507511.657, "dur": 0.406,
    "args": {
      "External id": 2487046,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507513.730, "dur": 0.335,
    "args": {
      "External id": 2487047,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507546.577, "dur": 33.716,
    "args": {
      "External id": 2487048,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507617.270, "dur": 101.211,
    "args": {
      "External id": 2487049,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507627.929, "dur": 6.275,
    "args": {
      "External id": 2487050,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507639.324, "dur": 10.876,
    "args": {
      "External id": 2487051,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507643.590, "dur": 6.187,
    "args": {
      "External id": 2487052,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 3851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507647.730, "dur": 0.530,
    "args": {
      "External id": 2487053,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507657.077, "dur": 24.586,
    "args": {
      "External id": 2487054,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507659.719, "dur": 0.484,
    "args": {
      "External id": 2487055,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507661.860, "dur": 0.478,
    "args": {
      "External id": 2487056,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507663.948, "dur": 0.608,
    "args": {
      "External id": 2487057,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507666.069, "dur": 0.343,
    "args": {
      "External id": 2487058,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507667.949, "dur": 0.453,
    "args": {
      "External id": 2487059,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507670.142, "dur": 0.494,
    "args": {
      "External id": 2487060,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507672.116, "dur": 2.130,
    "args": {
      "External id": 2487061,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507675.816, "dur": 0.426,
    "args": {
      "External id": 2487062,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507677.897, "dur": 0.338,
    "args": {
      "External id": 2487063,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507692.208, "dur": 19.137,
    "args": {
      "External id": 2487064,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507765.604, "dur": 108.681,
    "args": {
      "External id": 2487065,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 3864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507787.784, "dur": 83.149,
    "args": {
      "External id": 2487066,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3865, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507797.696, "dur": 69.411,
    "args": {
      "External id": 2487067,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 3866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507887.042, "dur": 1.780,
    "args": {
      "External id": 2487068,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3867, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657507962.408, "dur": 1685.941,
    "args": {
      "External id": 2487069,"Sequence number": 24740446, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3868
    }
  },
  {
    "ph": "f", "id": 202, "pid": 1336757, "tid": 1381170, "ts": 1514657507962.408,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508110.574, "dur": 106.247,
    "args": {
      "External id": 2487070,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 3869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508254.574, "dur": 43.662,
    "args": {
      "External id": 2487071,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 3870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508317.786, "dur": 51.109,
    "args": {
      "External id": 2487072,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 3871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508378.022, "dur": 34.099,
    "args": {
      "External id": 2487073,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508419.130, "dur": 47.673,
    "args": {
      "External id": 2487074,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508477.172, "dur": 29.767,
    "args": {
      "External id": 2487075,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 3874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508514.728, "dur": 64.290,
    "args": {
      "External id": 2487076,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 3875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508604.242, "dur": 24.858,
    "args": {
      "External id": 2487077,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 3876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508647.063, "dur": 30.483,
    "args": {
      "External id": 2487078,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508696.765, "dur": 21.442,
    "args": {
      "External id": 2487079,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508731.687, "dur": 15.292,
    "args": {
      "External id": 2487080,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508757.743, "dur": 32.222,
    "args": {
      "External id": 2487081,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508793.576, "dur": 34.836,
    "args": {
      "External id": 2487082,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508854.905, "dur": 209.495,
    "args": {
      "External id": 2487083,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 3882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508931.607, "dur": 5.513,
    "args": {
      "External id": 2487084,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657508939.432, "dur": 4.016,
    "args": {
      "External id": 2487085,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509101.079, "dur": 25.980,
    "args": {
      "External id": 2487086,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509138.055, "dur": 16.722,
    "args": {
      "External id": 2487087,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509164.573, "dur": 41.359,
    "args": {
      "External id": 2487088,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509212.840, "dur": 38.326,
    "args": {
      "External id": 2487089,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509258.044, "dur": 22.600,
    "args": {
      "External id": 2487090,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509288.869, "dur": 31.583,
    "args": {
      "External id": 2487091,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509327.036, "dur": 34.159,
    "args": {
      "External id": 2487092,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 3891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509374.925, "dur": 39.301,
    "args": {
      "External id": 2487093,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 3892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509432.208, "dur": 27.703,
    "args": {
      "External id": 2487094,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 3893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509476.460, "dur": 24.606,
    "args": {
      "External id": 2487095,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 3894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509518.863, "dur": 32.731,
    "args": {
      "External id": 2487096,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 3895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509570.353, "dur": 17.728,
    "args": {
      "External id": 2487097,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 3896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509601.928, "dur": 17.474,
    "args": {
      "External id": 2487098,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 3897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509689.259, "dur": 15.028,
    "args": {
      "External id": 2487099,"Record function id": 0, "Ev Idx": 3898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509692.426, "dur": 10.959,
    "args": {
      "External id": 2487100,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509696.738, "dur": 5.671,
    "args": {
      "External id": 2487101,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509698.375, "dur": 3.914,
    "args": {
      "External id": 2487102,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509708.019, "dur": 5.432,
    "args": {
      "External id": 2487103,"Record function id": 0, "Ev Idx": 3902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509709.681, "dur": 3.318,
    "args": {
      "External id": 2487104,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509710.429, "dur": 1.988,
    "args": {
      "External id": 2487105,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509711.269, "dur": 1.053,
    "args": {
      "External id": 2487106,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509716.607, "dur": 4.346,
    "args": {
      "External id": 2487107,"Record function id": 0, "Ev Idx": 3906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509717.992, "dur": 2.537,
    "args": {
      "External id": 2487108,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509718.608, "dur": 1.504,
    "args": {
      "External id": 2487109,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509718.984, "dur": 1.052,
    "args": {
      "External id": 2487110,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 3909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509724.136, "dur": 6.813,
    "args": {
      "External id": 2487111,"Record function id": 0, "Ev Idx": 3910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509725.548, "dur": 4.975,
    "args": {
      "External id": 2487112,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509726.045, "dur": 4.082,
    "args": {
      "External id": 2487113,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509727.133, "dur": 2.934,
    "args": {
      "External id": 2487114,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 3913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509734.048, "dur": 4.410,
    "args": {
      "External id": 2487115,"Record function id": 0, "Ev Idx": 3914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509735.268, "dur": 2.755,
    "args": {
      "External id": 2487116,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509735.934, "dur": 1.682,
    "args": {
      "External id": 2487117,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509736.612, "dur": 0.943,
    "args": {
      "External id": 2487118,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509741.667, "dur": 3.902,
    "args": {
      "External id": 2487119,"Record function id": 0, "Ev Idx": 3918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509742.864, "dur": 2.244,
    "args": {
      "External id": 2487120,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509743.480, "dur": 1.246,
    "args": {
      "External id": 2487121,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509744.017, "dur": 0.595,
    "args": {
      "External id": 2487122,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509748.832, "dur": 3.912,
    "args": {
      "External id": 2487123,"Record function id": 0, "Ev Idx": 3922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509750.095, "dur": 2.228,
    "args": {
      "External id": 2487124,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509750.693, "dur": 1.142,
    "args": {
      "External id": 2487125,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509751.127, "dur": 0.624,
    "args": {
      "External id": 2487126,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509755.952, "dur": 4.174,
    "args": {
      "External id": 2487127,"Record function id": 0, "Ev Idx": 3926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509757.264, "dur": 2.466,
    "args": {
      "External id": 2487128,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509758.086, "dur": 1.267,
    "args": {
      "External id": 2487129,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509758.618, "dur": 0.668,
    "args": {
      "External id": 2487130,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 3929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509763.173, "dur": 25.498,
    "args": {
      "External id": 2487131,"Record function id": 0, "Ev Idx": 3930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509764.446, "dur": 23.796,
    "args": {
      "External id": 2487132,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509765.153, "dur": 22.636,
    "args": {
      "External id": 2487133,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509786.819, "dur": 0.819,
    "args": {
      "External id": 2487134,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 3933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509792.689, "dur": 35155.892,
    "args": {
      "External id": 2487135,"Record function id": 0, "Sequence number": 24740445, "Fwd thread id": 1, "Ev Idx": 3934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509794.170, "dur": 35146.054,
    "args": {
      "External id": 2487136,"Sequence number": 24740445, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 3935
    }
  },
  {
    "ph": "f", "id": 203, "pid": 1336757, "tid": 1381170, "ts": 1514657509794.170,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.12)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509823.836, "dur": 36.605,
    "args": {
      "External id": 2487137,"Record function id": 0, "Ev Idx": 3936
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.12)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509868.264, "dur": 60.898,
    "args": {
      "External id": 2487138,"Record function id": 0, "Ev Idx": 3937
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.12)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657509936.691, "dur": 34996.105,
    "args": {
      "External id": 2487139,"Record function id": 0, "Ev Idx": 3938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657510077.023, "dur": 8.174,
    "args": {
      "External id": 2487140,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657510098.705, "dur": 4.979,
    "args": {
      "External id": 2487141,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657510118.629, "dur": 34033.850,
    "args": {
      "External id": 2487142,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657510131.841, "dur": 34012.671,
    "args": {
      "External id": 2487143,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 3942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657510176.912, "dur": 16.181,
    "args": {
      "External id": 2487144,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657510199.426, "dur": 33904.750,
    "args": {
      "External id": 2487145,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 3944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657510202.107, "dur": 33901.408,
    "args": {
      "External id": 2487146,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 3945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657510206.450, "dur": 5.691,
    "args": {
      "External id": 2487147,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657510214.017, "dur": 33885.983,
    "args": {
      "External id": 2487148,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 3947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544241.606, "dur": 9.017,
    "args": {
      "External id": 2487149,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 3948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544244.680, "dur": 5.623,
    "args": {
      "External id": 2487150,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544281.614, "dur": 353.649,
    "args": {
      "External id": 2487151,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 3950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544307.830, "dur": 321.784,
    "args": {
      "External id": 2487152,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 3951, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544318.662, "dur": 305.488,
    "args": {
      "External id": 2487153,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 3952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544655.447, "dur": 2.581,
    "args": {
      "External id": 2487154,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 3953, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544714.455, "dur": 7.030,
    "args": {
      "External id": 2487155,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544771.292, "dur": 1.419,
    "args": {
      "External id": 2487156,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544792.027, "dur": 1.060,
    "args": {
      "External id": 2487157,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544807.562, "dur": 1.072,
    "args": {
      "External id": 2487158,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544820.277, "dur": 0.849,
    "args": {
      "External id": 2487159,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544833.316, "dur": 1.116,
    "args": {
      "External id": 2487160,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544846.018, "dur": 0.842,
    "args": {
      "External id": 2487161,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544860.497, "dur": 2.274,
    "args": {
      "External id": 2487162,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544872.943, "dur": 1.127,
    "args": {
      "External id": 2487163,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657544962.746, "dur": 2775.915,
    "args": {
      "External id": 2487164,"Record function id": 0, "Ev Idx": 3963
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.11)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545017.940, "dur": 1022.598,
    "args": {
      "External id": 2487165,"Record function id": 0, "Ev Idx": 3964
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.11)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545033.968, "dur": 318.928,
    "args": {
      "External id": 2487166,"Record function id": 0, "Ev Idx": 3965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545121.336, "dur": 4.655,
    "args": {
      "External id": 2487167,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 3966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545129.299, "dur": 1.243,
    "args": {
      "External id": 2487168,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545132.237, "dur": 1.394,
    "args": {
      "External id": 2487169,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545135.166, "dur": 3.270,
    "args": {
      "External id": 2487170,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545139.790, "dur": 0.893,
    "args": {
      "External id": 2487171,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545142.171, "dur": 0.752,
    "args": {
      "External id": 2487172,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 3971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545144.390, "dur": 1.984,
    "args": {
      "External id": 2487173,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 3972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545147.816, "dur": 1.134,
    "args": {
      "External id": 2487174,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545150.340, "dur": 1.181,
    "args": {
      "External id": 2487175,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545152.823, "dur": 1.054,
    "args": {
      "External id": 2487176,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 3975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545170.523, "dur": 151.136,
    "args": {
      "External id": 2487177,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545186.199, "dur": 131.008,
    "args": {
      "External id": 2487178,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 3977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545204.680, "dur": 12.091,
    "args": {
      "External id": 2487179,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545219.933, "dur": 68.040,
    "args": {
      "External id": 2487180,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 3979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545223.168, "dur": 64.391,
    "args": {
      "External id": 2487181,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 3980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545227.304, "dur": 7.086,
    "args": {
      "External id": 2487182,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545236.215, "dur": 50.811,
    "args": {
      "External id": 2487183,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 3982
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.10", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545440.366, "dur": 591.001,
    "args": {
      "External id": 2487184,"Record function id": 0, "Ev Idx": 3983
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.10)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545455.245, "dur": 523.751,
    "args": {
      "External id": 2487185,"Record function id": 0, "Ev Idx": 3984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545517.763, "dur": 4.343,
    "args": {
      "External id": 2487186,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545554.377, "dur": 31.498,
    "args": {
      "External id": 2487187,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 3986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545559.368, "dur": 2.202,
    "args": {
      "External id": 2487188,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545564.035, "dur": 0.636,
    "args": {
      "External id": 2487189,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545566.330, "dur": 0.452,
    "args": {
      "External id": 2487190,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545568.402, "dur": 0.332,
    "args": {
      "External id": 2487191,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545570.210, "dur": 0.515,
    "args": {
      "External id": 2487192,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545572.905, "dur": 0.606,
    "args": {
      "External id": 2487193,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545574.732, "dur": 2.080,
    "args": {
      "External id": 2487194,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545578.426, "dur": 0.431,
    "args": {
      "External id": 2487195,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545580.751, "dur": 0.766,
    "args": {
      "External id": 2487196,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 3995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545596.550, "dur": 32.036,
    "args": {
      "External id": 2487197,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 3996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545662.351, "dur": 98.271,
    "args": {
      "External id": 2487198,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 3997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545673.590, "dur": 3.727,
    "args": {
      "External id": 2487199,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 3998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545682.288, "dur": 10.252,
    "args": {
      "External id": 2487200,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 3999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545686.496, "dur": 5.641,
    "args": {
      "External id": 2487201,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545690.168, "dur": 0.625,
    "args": {
      "External id": 2487202,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545699.694, "dur": 24.628,
    "args": {
      "External id": 2487203,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545702.023, "dur": 0.961,
    "args": {
      "External id": 2487204,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545704.616, "dur": 0.787,
    "args": {
      "External id": 2487205,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545707.112, "dur": 0.432,
    "args": {
      "External id": 2487206,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545709.156, "dur": 2.214,
    "args": {
      "External id": 2487207,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545713.111, "dur": 0.340,
    "args": {
      "External id": 2487208,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545715.015, "dur": 0.384,
    "args": {
      "External id": 2487209,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545717.020, "dur": 0.408,
    "args": {
      "External id": 2487210,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545718.985, "dur": 0.421,
    "args": {
      "External id": 2487211,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545721.067, "dur": 0.618,
    "args": {
      "External id": 2487212,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545734.647, "dur": 18.505,
    "args": {
      "External id": 2487213,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545810.948, "dur": 106.417,
    "args": {
      "External id": 2487214,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545832.564, "dur": 81.628,
    "args": {
      "External id": 2487215,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4014, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545842.093, "dur": 67.335,
    "args": {
      "External id": 2487216,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657545930.139, "dur": 1.713,
    "args": {
      "External id": 2487217,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4016, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657546048.285, "dur": 1670.919,
    "args": {
      "External id": 2487218,"Sequence number": 24740444, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4017
    }
  },
  {
    "ph": "f", "id": 204, "pid": 1336757, "tid": 1381170, "ts": 1514657546048.285,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657546161.824, "dur": 106.035,
    "args": {
      "External id": 2487219,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657546305.682, "dur": 42.576,
    "args": {
      "External id": 2487220,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657546365.870, "dur": 49.834,
    "args": {
      "External id": 2487221,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657546425.026, "dur": 33.520,
    "args": {
      "External id": 2487222,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657546467.558, "dur": 47.291,
    "args": {
      "External id": 2487223,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657546523.227, "dur": 47.592,
    "args": {
      "External id": 2487224,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657546581.420, "dur": 46.267,
    "args": {
      "External id": 2487225,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657546651.182, "dur": 27.099,
    "args": {
      "External id": 2487226,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657546697.573, "dur": 30.864,
    "args": {
      "External id": 2487227,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657546747.658, "dur": 21.750,
    "args": {
      "External id": 2487228,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657546782.205, "dur": 16.808,
    "args": {
      "External id": 2487229,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657546808.209, "dur": 29.744,
    "args": {
      "External id": 2487230,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657546841.560, "dur": 35.517,
    "args": {
      "External id": 2487231,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657546904.715, "dur": 233.838,
    "args": {
      "External id": 2487232,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547028.047, "dur": 8.155,
    "args": {
      "External id": 2487233,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547039.778, "dur": 2.224,
    "args": {
      "External id": 2487234,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547174.677, "dur": 26.375,
    "args": {
      "External id": 2487235,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547212.022, "dur": 16.147,
    "args": {
      "External id": 2487236,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547236.663, "dur": 40.731,
    "args": {
      "External id": 2487237,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547284.751, "dur": 36.819,
    "args": {
      "External id": 2487238,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547330.688, "dur": 22.916,
    "args": {
      "External id": 2487239,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547359.926, "dur": 31.433,
    "args": {
      "External id": 2487240,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547398.599, "dur": 20.815,
    "args": {
      "External id": 2487241,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547438.696, "dur": 42.827,
    "args": {
      "External id": 2487242,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547499.414, "dur": 24.134,
    "args": {
      "External id": 2487243,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547559.232, "dur": 28.289,
    "args": {
      "External id": 2487244,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547608.104, "dur": 18.278,
    "args": {
      "External id": 2487245,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547643.815, "dur": 16.309,
    "args": {
      "External id": 2487246,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547671.683, "dur": 17.932,
    "args": {
      "External id": 2487247,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547761.221, "dur": 15.338,
    "args": {
      "External id": 2487248,"Record function id": 0, "Ev Idx": 4047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547764.531, "dur": 11.203,
    "args": {
      "External id": 2487249,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547768.918, "dur": 5.798,
    "args": {
      "External id": 2487250,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547770.461, "dur": 4.125,
    "args": {
      "External id": 2487251,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547780.649, "dur": 5.354,
    "args": {
      "External id": 2487252,"Record function id": 0, "Ev Idx": 4051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547782.041, "dur": 3.491,
    "args": {
      "External id": 2487253,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547782.863, "dur": 2.117,
    "args": {
      "External id": 2487254,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547783.728, "dur": 1.149,
    "args": {
      "External id": 2487255,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547789.392, "dur": 6.039,
    "args": {
      "External id": 2487256,"Record function id": 0, "Ev Idx": 4055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547790.533, "dur": 4.471,
    "args": {
      "External id": 2487257,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547791.282, "dur": 3.263,
    "args": {
      "External id": 2487258,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547791.684, "dur": 2.793,
    "args": {
      "External id": 2487259,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547798.673, "dur": 4.565,
    "args": {
      "External id": 2487260,"Record function id": 0, "Ev Idx": 4059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547799.905, "dur": 2.938,
    "args": {
      "External id": 2487261,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547800.716, "dur": 1.718,
    "args": {
      "External id": 2487262,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547801.369, "dur": 0.999,
    "args": {
      "External id": 2487263,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547806.290, "dur": 3.949,
    "args": {
      "External id": 2487264,"Record function id": 0, "Ev Idx": 4063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547807.376, "dur": 2.463,
    "args": {
      "External id": 2487265,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547808.153, "dur": 1.307,
    "args": {
      "External id": 2487266,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547808.670, "dur": 0.726,
    "args": {
      "External id": 2487267,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547813.417, "dur": 3.627,
    "args": {
      "External id": 2487268,"Record function id": 0, "Ev Idx": 4067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547814.334, "dur": 2.264,
    "args": {
      "External id": 2487269,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547814.853, "dur": 1.186,
    "args": {
      "External id": 2487270,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547815.290, "dur": 0.660,
    "args": {
      "External id": 2487271,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547820.319, "dur": 3.627,
    "args": {
      "External id": 2487272,"Record function id": 0, "Ev Idx": 4071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547821.414, "dur": 2.107,
    "args": {
      "External id": 2487273,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547821.886, "dur": 1.126,
    "args": {
      "External id": 2487274,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547822.179, "dur": 0.741,
    "args": {
      "External id": 2487275,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547827.012, "dur": 3.357,
    "args": {
      "External id": 2487276,"Record function id": 0, "Ev Idx": 4075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547828.097, "dur": 1.861,
    "args": {
      "External id": 2487277,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547828.567, "dur": 0.988,
    "args": {
      "External id": 2487278,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547828.941, "dur": 0.550,
    "args": {
      "External id": 2487279,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547833.410, "dur": 3.873,
    "args": {
      "External id": 2487280,"Record function id": 0, "Ev Idx": 4079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547834.744, "dur": 2.112,
    "args": {
      "External id": 2487281,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547835.225, "dur": 1.249,
    "args": {
      "External id": 2487282,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547835.709, "dur": 0.668,
    "args": {
      "External id": 2487283,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547841.191, "dur": 36223.862,
    "args": {
      "External id": 2487284,"Record function id": 0, "Sequence number": 24740443, "Fwd thread id": 1, "Ev Idx": 4083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547842.312, "dur": 36213.565,
    "args": {
      "External id": 2487285,"Sequence number": 24740443, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4084
    }
  },
  {
    "ph": "f", "id": 205, "pid": 1336757, "tid": 1381170, "ts": 1514657547842.312,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.11)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547871.316, "dur": 35.205,
    "args": {
      "External id": 2487286,"Record function id": 0, "Ev Idx": 4085
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.11)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657547913.802, "dur": 61.597,
    "args": {
      "External id": 2487287,"Record function id": 0, "Ev Idx": 4086
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.11)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657548015.402, "dur": 36031.572,
    "args": {
      "External id": 2487288,"Record function id": 0, "Ev Idx": 4087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657548104.584, "dur": 6.849,
    "args": {
      "External id": 2487289,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657548121.548, "dur": 7.311,
    "args": {
      "External id": 2487290,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657548143.533, "dur": 35087.677,
    "args": {
      "External id": 2487291,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657548158.142, "dur": 35064.224,
    "args": {
      "External id": 2487292,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657548225.466, "dur": 14.087,
    "args": {
      "External id": 2487293,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657548246.091, "dur": 34935.406,
    "args": {
      "External id": 2487294,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657548248.949, "dur": 34931.668,
    "args": {
      "External id": 2487295,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657548252.778, "dur": 4.839,
    "args": {
      "External id": 2487296,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657548279.908, "dur": 34897.039,
    "args": {
      "External id": 2487297,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657583319.397, "dur": 9.206,
    "args": {
      "External id": 2487298,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657583322.756, "dur": 5.537,
    "args": {
      "External id": 2487299,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657583358.131, "dur": 362.506,
    "args": {
      "External id": 2487300,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657583383.091, "dur": 332.318,
    "args": {
      "External id": 2487301,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4100, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657583393.815, "dur": 315.867,
    "args": {
      "External id": 2487302,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657583740.430, "dur": 2.481,
    "args": {
      "External id": 2487303,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4102, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657583799.525, "dur": 6.768,
    "args": {
      "External id": 2487304,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657583852.025, "dur": 1.408,
    "args": {
      "External id": 2487305,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657583869.708, "dur": 3.270,
    "args": {
      "External id": 2487306,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657583884.768, "dur": 0.993,
    "args": {
      "External id": 2487307,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657583897.305, "dur": 1.163,
    "args": {
      "External id": 2487308,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657583909.622, "dur": 0.925,
    "args": {
      "External id": 2487309,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657583926.453, "dur": 3.009,
    "args": {
      "External id": 2487310,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657583940.653, "dur": 2.380,
    "args": {
      "External id": 2487311,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657583953.198, "dur": 0.996,
    "args": {
      "External id": 2487312,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584081.116, "dur": 2767.954,
    "args": {
      "External id": 2487313,"Record function id": 0, "Ev Idx": 4112
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.10)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584101.908, "dur": 1032.103,
    "args": {
      "External id": 2487314,"Record function id": 0, "Ev Idx": 4113
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.10)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584116.058, "dur": 313.839,
    "args": {
      "External id": 2487315,"Record function id": 0, "Ev Idx": 4114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584201.167, "dur": 4.912,
    "args": {
      "External id": 2487316,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584209.280, "dur": 1.198,
    "args": {
      "External id": 2487317,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584212.172, "dur": 3.045,
    "args": {
      "External id": 2487318,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584216.903, "dur": 1.067,
    "args": {
      "External id": 2487319,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584219.721, "dur": 1.184,
    "args": {
      "External id": 2487320,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584222.514, "dur": 1.106,
    "args": {
      "External id": 2487321,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584225.411, "dur": 1.668,
    "args": {
      "External id": 2487322,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584228.539, "dur": 0.976,
    "args": {
      "External id": 2487323,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584230.918, "dur": 0.959,
    "args": {
      "External id": 2487324,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584233.196, "dur": 0.884,
    "args": {
      "External id": 2487325,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584251.418, "dur": 148.555,
    "args": {
      "External id": 2487326,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584266.887, "dur": 128.913,
    "args": {
      "External id": 2487327,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584283.907, "dur": 13.030,
    "args": {
      "External id": 2487328,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584300.546, "dur": 66.010,
    "args": {
      "External id": 2487329,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584303.402, "dur": 62.784,
    "args": {
      "External id": 2487330,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584307.650, "dur": 5.141,
    "args": {
      "External id": 2487331,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584315.094, "dur": 50.498,
    "args": {
      "External id": 2487332,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4131
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.9", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584515.321, "dur": 610.072,
    "args": {
      "External id": 2487333,"Record function id": 0, "Ev Idx": 4132
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.9)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584548.997, "dur": 563.465,
    "args": {
      "External id": 2487334,"Record function id": 0, "Ev Idx": 4133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584616.921, "dur": 5.696,
    "args": {
      "External id": 2487335,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584638.544, "dur": 34.545,
    "args": {
      "External id": 2487336,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584643.399, "dur": 1.622,
    "args": {
      "External id": 2487337,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584647.310, "dur": 0.688,
    "args": {
      "External id": 2487338,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584650.067, "dur": 0.446,
    "args": {
      "External id": 2487339,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584652.492, "dur": 0.633,
    "args": {
      "External id": 2487340,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584654.654, "dur": 0.367,
    "args": {
      "External id": 2487341,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584656.478, "dur": 2.569,
    "args": {
      "External id": 2487342,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584665.300, "dur": 0.623,
    "args": {
      "External id": 2487343,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584667.493, "dur": 0.488,
    "args": {
      "External id": 2487344,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584669.457, "dur": 0.430,
    "args": {
      "External id": 2487345,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584683.647, "dur": 33.090,
    "args": {
      "External id": 2487346,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584748.429, "dur": 101.226,
    "args": {
      "External id": 2487347,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584759.120, "dur": 3.029,
    "args": {
      "External id": 2487348,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584766.984, "dur": 10.441,
    "args": {
      "External id": 2487349,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584771.350, "dur": 5.649,
    "args": {
      "External id": 2487350,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584775.130, "dur": 0.661,
    "args": {
      "External id": 2487351,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584787.245, "dur": 26.351,
    "args": {
      "External id": 2487352,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584789.440, "dur": 0.466,
    "args": {
      "External id": 2487353,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584792.023, "dur": 0.419,
    "args": {
      "External id": 2487354,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584794.086, "dur": 2.045,
    "args": {
      "External id": 2487355,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584797.346, "dur": 0.390,
    "args": {
      "External id": 2487356,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584799.171, "dur": 0.484,
    "args": {
      "External id": 2487357,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584801.297, "dur": 0.371,
    "args": {
      "External id": 2487358,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584803.281, "dur": 0.343,
    "args": {
      "External id": 2487359,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584805.142, "dur": 0.469,
    "args": {
      "External id": 2487360,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584807.082, "dur": 0.523,
    "args": {
      "External id": 2487361,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584823.870, "dur": 17.769,
    "args": {
      "External id": 2487362,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584893.169, "dur": 148.097,
    "args": {
      "External id": 2487363,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584913.594, "dur": 123.047,
    "args": {
      "External id": 2487364,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4163, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657584924.336, "dur": 107.520,
    "args": {
      "External id": 2487365,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657585056.532, "dur": 2.247,
    "args": {
      "External id": 2487366,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4165, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657585141.481, "dur": 1686.290,
    "args": {
      "External id": 2487367,"Sequence number": 24740442, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4166
    }
  },
  {
    "ph": "f", "id": 206, "pid": 1336757, "tid": 1381170, "ts": 1514657585141.481,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657585251.939, "dur": 105.731,
    "args": {
      "External id": 2487368,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657585396.148, "dur": 44.096,
    "args": {
      "External id": 2487369,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657585457.742, "dur": 49.827,
    "args": {
      "External id": 2487370,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657585518.770, "dur": 53.790,
    "args": {
      "External id": 2487371,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657585582.607, "dur": 49.803,
    "args": {
      "External id": 2487372,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657585641.063, "dur": 28.345,
    "args": {
      "External id": 2487373,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657585677.201, "dur": 48.098,
    "args": {
      "External id": 2487374,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657585751.262, "dur": 26.442,
    "args": {
      "External id": 2487375,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657585805.800, "dur": 32.759,
    "args": {
      "External id": 2487376,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657585860.084, "dur": 22.373,
    "args": {
      "External id": 2487377,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657585897.565, "dur": 16.371,
    "args": {
      "External id": 2487378,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657585923.203, "dur": 32.176,
    "args": {
      "External id": 2487379,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657585958.312, "dur": 72.507,
    "args": {
      "External id": 2487380,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586065.574, "dur": 172.568,
    "args": {
      "External id": 2487381,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586141.829, "dur": 6.542,
    "args": {
      "External id": 2487382,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586150.538, "dur": 2.293,
    "args": {
      "External id": 2487383,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586272.595, "dur": 24.349,
    "args": {
      "External id": 2487384,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586307.605, "dur": 16.937,
    "args": {
      "External id": 2487385,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586333.266, "dur": 39.392,
    "args": {
      "External id": 2487386,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586379.607, "dur": 37.202,
    "args": {
      "External id": 2487387,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586425.652, "dur": 22.714,
    "args": {
      "External id": 2487388,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586453.909, "dur": 31.807,
    "args": {
      "External id": 2487389,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586492.314, "dur": 54.977,
    "args": {
      "External id": 2487390,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586567.631, "dur": 43.603,
    "args": {
      "External id": 2487391,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586633.788, "dur": 24.905,
    "args": {
      "External id": 2487392,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586674.886, "dur": 28.665,
    "args": {
      "External id": 2487393,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586717.775, "dur": 18.352,
    "args": {
      "External id": 2487394,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586754.129, "dur": 16.530,
    "args": {
      "External id": 2487395,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586783.107, "dur": 17.155,
    "args": {
      "External id": 2487396,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586871.295, "dur": 14.806,
    "args": {
      "External id": 2487397,"Record function id": 0, "Ev Idx": 4196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586874.444, "dur": 10.871,
    "args": {
      "External id": 2487398,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586878.542, "dur": 5.883,
    "args": {
      "External id": 2487399,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586880.204, "dur": 4.074,
    "args": {
      "External id": 2487400,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586889.872, "dur": 5.830,
    "args": {
      "External id": 2487401,"Record function id": 0, "Ev Idx": 4200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586891.573, "dur": 3.676,
    "args": {
      "External id": 2487402,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586892.508, "dur": 2.261,
    "args": {
      "External id": 2487403,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586893.715, "dur": 0.973,
    "args": {
      "External id": 2487404,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586899.385, "dur": 4.449,
    "args": {
      "External id": 2487405,"Record function id": 0, "Ev Idx": 4204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586900.759, "dur": 2.642,
    "args": {
      "External id": 2487406,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586901.316, "dur": 1.654,
    "args": {
      "External id": 2487407,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586901.646, "dur": 1.256,
    "args": {
      "External id": 2487408,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586907.034, "dur": 5.559,
    "args": {
      "External id": 2487409,"Record function id": 0, "Ev Idx": 4208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586908.345, "dur": 3.825,
    "args": {
      "External id": 2487410,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586908.918, "dur": 2.780,
    "args": {
      "External id": 2487411,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586909.224, "dur": 2.413,
    "args": {
      "External id": 2487412,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586915.847, "dur": 3.862,
    "args": {
      "External id": 2487413,"Record function id": 0, "Ev Idx": 4212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586917.085, "dur": 2.218,
    "args": {
      "External id": 2487414,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586917.721, "dur": 1.154,
    "args": {
      "External id": 2487415,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586918.025, "dur": 0.787,
    "args": {
      "External id": 2487416,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586922.820, "dur": 4.128,
    "args": {
      "External id": 2487417,"Record function id": 0, "Ev Idx": 4216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586924.096, "dur": 2.414,
    "args": {
      "External id": 2487418,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586924.556, "dur": 1.426,
    "args": {
      "External id": 2487419,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586925.006, "dur": 0.859,
    "args": {
      "External id": 2487420,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586930.388, "dur": 3.823,
    "args": {
      "External id": 2487421,"Record function id": 0, "Ev Idx": 4220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586931.517, "dur": 2.272,
    "args": {
      "External id": 2487422,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586932.346, "dur": 0.920,
    "args": {
      "External id": 2487423,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586932.635, "dur": 0.558,
    "args": {
      "External id": 2487424,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586937.301, "dur": 3.608,
    "args": {
      "External id": 2487425,"Record function id": 0, "Ev Idx": 4224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586938.459, "dur": 2.036,
    "args": {
      "External id": 2487426,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586938.966, "dur": 1.100,
    "args": {
      "External id": 2487427,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586939.230, "dur": 0.718,
    "args": {
      "External id": 2487428,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586943.965, "dur": 4.612,
    "args": {
      "External id": 2487429,"Record function id": 0, "Ev Idx": 4228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586945.377, "dur": 2.766,
    "args": {
      "External id": 2487430,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586946.090, "dur": 1.657,
    "args": {
      "External id": 2487431,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586946.969, "dur": 0.653,
    "args": {
      "External id": 2487432,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586952.561, "dur": 36007.295,
    "args": {
      "External id": 2487433,"Record function id": 0, "Sequence number": 24740441, "Fwd thread id": 1, "Ev Idx": 4232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657586954.065, "dur": 35997.120,
    "args": {
      "External id": 2487434,"Sequence number": 24740441, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4233
    }
  },
  {
    "ph": "f", "id": 207, "pid": 1336757, "tid": 1381170, "ts": 1514657586954.065,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.10)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657587013.467, "dur": 40.627,
    "args": {
      "External id": 2487435,"Record function id": 0, "Ev Idx": 4234
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.10)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657587062.844, "dur": 63.580,
    "args": {
      "External id": 2487436,"Record function id": 0, "Ev Idx": 4235
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.10)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657587132.197, "dur": 35811.277,
    "args": {
      "External id": 2487437,"Record function id": 0, "Ev Idx": 4236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657587218.315, "dur": 6.822,
    "args": {
      "External id": 2487438,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657587235.322, "dur": 5.135,
    "args": {
      "External id": 2487439,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657587256.010, "dur": 34855.803,
    "args": {
      "External id": 2487440,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657587269.645, "dur": 34833.232,
    "args": {
      "External id": 2487441,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657587324.979, "dur": 16.191,
    "args": {
      "External id": 2487442,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657587347.879, "dur": 34715.106,
    "args": {
      "External id": 2487443,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657587350.809, "dur": 34711.133,
    "args": {
      "External id": 2487444,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657587354.890, "dur": 4.902,
    "args": {
      "External id": 2487445,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657587361.807, "dur": 34696.405,
    "args": {
      "External id": 2487446,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622200.981, "dur": 9.507,
    "args": {
      "External id": 2487447,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622204.329, "dur": 5.749,
    "args": {
      "External id": 2487448,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622239.967, "dur": 406.167,
    "args": {
      "External id": 2487449,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622265.563, "dur": 375.404,
    "args": {
      "External id": 2487450,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4249, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622276.508, "dur": 357.617,
    "args": {
      "External id": 2487451,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622667.617, "dur": 2.489,
    "args": {
      "External id": 2487452,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4251, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622731.220, "dur": 6.599,
    "args": {
      "External id": 2487453,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622782.816, "dur": 1.633,
    "args": {
      "External id": 2487454,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622801.593, "dur": 1.114,
    "args": {
      "External id": 2487455,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622815.092, "dur": 3.207,
    "args": {
      "External id": 2487456,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622831.165, "dur": 1.266,
    "args": {
      "External id": 2487457,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622842.858, "dur": 1.071,
    "args": {
      "External id": 2487458,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622855.410, "dur": 0.948,
    "args": {
      "External id": 2487459,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622868.562, "dur": 3.390,
    "args": {
      "External id": 2487460,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622882.521, "dur": 0.858,
    "args": {
      "External id": 2487461,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657622974.789, "dur": 2824.965,
    "args": {
      "External id": 2487462,"Record function id": 0, "Ev Idx": 4261
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.9)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623029.814, "dur": 1052.256,
    "args": {
      "External id": 2487463,"Record function id": 0, "Ev Idx": 4262
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.9)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623045.700, "dur": 334.625,
    "args": {
      "External id": 2487464,"Record function id": 0, "Ev Idx": 4263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623133.526, "dur": 4.649,
    "args": {
      "External id": 2487465,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623141.675, "dur": 1.531,
    "args": {
      "External id": 2487466,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623144.969, "dur": 1.117,
    "args": {
      "External id": 2487467,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623147.585, "dur": 1.878,
    "args": {
      "External id": 2487468,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623150.974, "dur": 1.413,
    "args": {
      "External id": 2487469,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623153.974, "dur": 2.948,
    "args": {
      "External id": 2487470,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623158.669, "dur": 1.434,
    "args": {
      "External id": 2487471,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623161.544, "dur": 1.310,
    "args": {
      "External id": 2487472,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623164.209, "dur": 1.606,
    "args": {
      "External id": 2487473,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623167.243, "dur": 1.269,
    "args": {
      "External id": 2487474,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623186.245, "dur": 159.176,
    "args": {
      "External id": 2487475,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623201.973, "dur": 139.200,
    "args": {
      "External id": 2487476,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623224.178, "dur": 12.608,
    "args": {
      "External id": 2487477,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623240.382, "dur": 70.850,
    "args": {
      "External id": 2487478,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623243.322, "dur": 67.493,
    "args": {
      "External id": 2487479,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623247.342, "dur": 6.168,
    "args": {
      "External id": 2487480,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623257.346, "dur": 52.762,
    "args": {
      "External id": 2487481,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4280
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.8", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623472.056, "dur": 602.022,
    "args": {
      "External id": 2487482,"Record function id": 0, "Ev Idx": 4281
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.8)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623488.857, "dur": 572.307,
    "args": {
      "External id": 2487483,"Record function id": 0, "Ev Idx": 4282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623568.862, "dur": 6.070,
    "args": {
      "External id": 2487484,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623590.162, "dur": 36.361,
    "args": {
      "External id": 2487485,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623595.471, "dur": 3.815,
    "args": {
      "External id": 2487486,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623601.740, "dur": 0.621,
    "args": {
      "External id": 2487487,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623604.029, "dur": 0.623,
    "args": {
      "External id": 2487488,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623606.260, "dur": 0.569,
    "args": {
      "External id": 2487489,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623608.517, "dur": 0.449,
    "args": {
      "External id": 2487490,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623610.851, "dur": 0.596,
    "args": {
      "External id": 2487491,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623613.005, "dur": 0.440,
    "args": {
      "External id": 2487492,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623615.186, "dur": 0.406,
    "args": {
      "External id": 2487493,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623617.151, "dur": 2.335,
    "args": {
      "External id": 2487494,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623636.804, "dur": 32.486,
    "args": {
      "External id": 2487495,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623700.369, "dur": 103.725,
    "args": {
      "External id": 2487496,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623711.497, "dur": 3.431,
    "args": {
      "External id": 2487497,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623720.068, "dur": 11.320,
    "args": {
      "External id": 2487498,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623724.814, "dur": 6.165,
    "args": {
      "External id": 2487499,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623728.646, "dur": 0.575,
    "args": {
      "External id": 2487500,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623738.406, "dur": 28.926,
    "args": {
      "External id": 2487501,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623740.660, "dur": 0.490,
    "args": {
      "External id": 2487502,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623743.061, "dur": 0.600,
    "args": {
      "External id": 2487503,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623745.394, "dur": 0.725,
    "args": {
      "External id": 2487504,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623747.867, "dur": 0.601,
    "args": {
      "External id": 2487505,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623750.075, "dur": 0.549,
    "args": {
      "External id": 2487506,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623752.507, "dur": 2.058,
    "args": {
      "External id": 2487507,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623756.217, "dur": 0.622,
    "args": {
      "External id": 2487508,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623758.965, "dur": 0.375,
    "args": {
      "External id": 2487509,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623760.723, "dur": 0.541,
    "args": {
      "External id": 2487510,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623777.363, "dur": 18.791,
    "args": {
      "External id": 2487511,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623849.198, "dur": 108.515,
    "args": {
      "External id": 2487512,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623870.610, "dur": 83.606,
    "args": {
      "External id": 2487513,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4312, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623880.769, "dur": 69.253,
    "args": {
      "External id": 2487514,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657623969.839, "dur": 1.823,
    "args": {
      "External id": 2487515,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4314, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657624090.533, "dur": 1688.647,
    "args": {
      "External id": 2487516,"Sequence number": 24740440, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4315
    }
  },
  {
    "ph": "f", "id": 208, "pid": 1336757, "tid": 1381170, "ts": 1514657624090.533,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657624203.480, "dur": 108.755,
    "args": {
      "External id": 2487517,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657624351.283, "dur": 40.203,
    "args": {
      "External id": 2487518,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657624410.497, "dur": 51.554,
    "args": {
      "External id": 2487519,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657624471.404, "dur": 32.961,
    "args": {
      "External id": 2487520,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657624511.645, "dur": 65.531,
    "args": {
      "External id": 2487521,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657624590.958, "dur": 32.648,
    "args": {
      "External id": 2487522,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657624632.030, "dur": 43.482,
    "args": {
      "External id": 2487523,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657624699.462, "dur": 25.956,
    "args": {
      "External id": 2487524,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657624743.844, "dur": 29.856,
    "args": {
      "External id": 2487525,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657624792.376, "dur": 19.883,
    "args": {
      "External id": 2487526,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657624826.170, "dur": 17.622,
    "args": {
      "External id": 2487527,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657624852.036, "dur": 30.134,
    "args": {
      "External id": 2487528,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657624885.457, "dur": 33.861,
    "args": {
      "External id": 2487529,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657624947.569, "dur": 219.358,
    "args": {
      "External id": 2487530,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625063.641, "dur": 8.719,
    "args": {
      "External id": 2487531,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625074.837, "dur": 2.402,
    "args": {
      "External id": 2487532,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625201.061, "dur": 26.870,
    "args": {
      "External id": 2487533,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625239.251, "dur": 16.858,
    "args": {
      "External id": 2487534,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625265.373, "dur": 41.880,
    "args": {
      "External id": 2487535,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625314.114, "dur": 37.830,
    "args": {
      "External id": 2487536,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625358.564, "dur": 22.651,
    "args": {
      "External id": 2487537,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625417.988, "dur": 44.740,
    "args": {
      "External id": 2487538,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625475.175, "dur": 23.541,
    "args": {
      "External id": 2487539,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625507.160, "dur": 50.578,
    "args": {
      "External id": 2487540,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625584.133, "dur": 26.387,
    "args": {
      "External id": 2487541,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625628.114, "dur": 25.982,
    "args": {
      "External id": 2487542,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625670.405, "dur": 17.350,
    "args": {
      "External id": 2487543,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625703.662, "dur": 16.512,
    "args": {
      "External id": 2487544,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625734.162, "dur": 16.960,
    "args": {
      "External id": 2487545,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625822.221, "dur": 15.040,
    "args": {
      "External id": 2487546,"Record function id": 0, "Ev Idx": 4345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625825.311, "dur": 11.000,
    "args": {
      "External id": 2487547,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625829.484, "dur": 5.939,
    "args": {
      "External id": 2487548,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625831.297, "dur": 4.008,
    "args": {
      "External id": 2487549,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625841.237, "dur": 5.338,
    "args": {
      "External id": 2487550,"Record function id": 0, "Ev Idx": 4349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625842.766, "dur": 3.358,
    "args": {
      "External id": 2487551,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625843.690, "dur": 1.978,
    "args": {
      "External id": 2487552,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625844.548, "dur": 1.022,
    "args": {
      "External id": 2487553,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625849.793, "dur": 4.877,
    "args": {
      "External id": 2487554,"Record function id": 0, "Ev Idx": 4353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625851.350, "dur": 2.891,
    "args": {
      "External id": 2487555,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625851.910, "dur": 1.893,
    "args": {
      "External id": 2487556,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625852.616, "dur": 1.123,
    "args": {
      "External id": 2487557,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625857.840, "dur": 4.095,
    "args": {
      "External id": 2487558,"Record function id": 0, "Ev Idx": 4357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625859.053, "dur": 2.450,
    "args": {
      "External id": 2487559,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625859.823, "dur": 1.285,
    "args": {
      "External id": 2487560,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625860.350, "dur": 0.690,
    "args": {
      "External id": 2487561,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625865.036, "dur": 6.170,
    "args": {
      "External id": 2487562,"Record function id": 0, "Ev Idx": 4361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625866.105, "dur": 4.685,
    "args": {
      "External id": 2487563,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625866.815, "dur": 3.539,
    "args": {
      "External id": 2487564,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625867.183, "dur": 3.111,
    "args": {
      "External id": 2487565,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625874.406, "dur": 3.973,
    "args": {
      "External id": 2487566,"Record function id": 0, "Ev Idx": 4365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625875.390, "dur": 2.499,
    "args": {
      "External id": 2487567,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625876.042, "dur": 1.456,
    "args": {
      "External id": 2487568,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625876.564, "dur": 0.851,
    "args": {
      "External id": 2487569,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625881.640, "dur": 3.506,
    "args": {
      "External id": 2487570,"Record function id": 0, "Ev Idx": 4369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625882.587, "dur": 2.127,
    "args": {
      "External id": 2487571,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625883.075, "dur": 1.120,
    "args": {
      "External id": 2487572,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625883.456, "dur": 0.676,
    "args": {
      "External id": 2487573,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625888.188, "dur": 3.407,
    "args": {
      "External id": 2487574,"Record function id": 0, "Ev Idx": 4373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625889.180, "dur": 1.969,
    "args": {
      "External id": 2487575,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625889.780, "dur": 0.939,
    "args": {
      "External id": 2487576,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625890.045, "dur": 0.561,
    "args": {
      "External id": 2487577,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625894.701, "dur": 3.713,
    "args": {
      "External id": 2487578,"Record function id": 0, "Ev Idx": 4377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625895.697, "dur": 2.233,
    "args": {
      "External id": 2487579,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625896.305, "dur": 1.226,
    "args": {
      "External id": 2487580,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625896.845, "dur": 0.568,
    "args": {
      "External id": 2487581,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625902.236, "dur": 36732.648,
    "args": {
      "External id": 2487582,"Record function id": 0, "Sequence number": 24740439, "Fwd thread id": 1, "Ev Idx": 4381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625903.511, "dur": 36722.583,
    "args": {
      "External id": 2487583,"Sequence number": 24740439, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4382
    }
  },
  {
    "ph": "f", "id": 209, "pid": 1336757, "tid": 1381170, "ts": 1514657625903.511,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.9)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625930.359, "dur": 37.019,
    "args": {
      "External id": 2487584,"Record function id": 0, "Ev Idx": 4383
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.9)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657625974.713, "dur": 115.650,
    "args": {
      "External id": 2487585,"Record function id": 0, "Ev Idx": 4384
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.9)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657626099.178, "dur": 36519.248,
    "args": {
      "External id": 2487586,"Record function id": 0, "Ev Idx": 4385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657626187.965, "dur": 7.104,
    "args": {
      "External id": 2487587,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657626205.580, "dur": 4.914,
    "args": {
      "External id": 2487588,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657626224.568, "dur": 35586.593,
    "args": {
      "External id": 2487589,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657626238.338, "dur": 35564.320,
    "args": {
      "External id": 2487590,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657626286.895, "dur": 14.454,
    "args": {
      "External id": 2487591,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657626307.691, "dur": 35454.364,
    "args": {
      "External id": 2487592,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657626310.365, "dur": 35451.029,
    "args": {
      "External id": 2487593,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657626314.216, "dur": 7.231,
    "args": {
      "External id": 2487594,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657626323.340, "dur": 35434.126,
    "args": {
      "External id": 2487595,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657661901.356, "dur": 9.531,
    "args": {
      "External id": 2487596,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657661904.501, "dur": 6.033,
    "args": {
      "External id": 2487597,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657661939.774, "dur": 364.965,
    "args": {
      "External id": 2487598,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657661966.372, "dur": 333.376,
    "args": {
      "External id": 2487599,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4398, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657661977.856, "dur": 315.872,
    "args": {
      "External id": 2487600,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662324.445, "dur": 2.751,
    "args": {
      "External id": 2487601,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4400, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662386.953, "dur": 6.950,
    "args": {
      "External id": 2487602,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662438.369, "dur": 1.586,
    "args": {
      "External id": 2487603,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662456.894, "dur": 1.253,
    "args": {
      "External id": 2487604,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662470.043, "dur": 3.059,
    "args": {
      "External id": 2487605,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662485.795, "dur": 1.042,
    "args": {
      "External id": 2487606,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662497.265, "dur": 1.164,
    "args": {
      "External id": 2487607,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662509.757, "dur": 1.241,
    "args": {
      "External id": 2487608,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662522.221, "dur": 2.958,
    "args": {
      "External id": 2487609,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662554.720, "dur": 2.244,
    "args": {
      "External id": 2487610,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662650.121, "dur": 2752.065,
    "args": {
      "External id": 2487611,"Record function id": 0, "Ev Idx": 4410
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.8)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662669.063, "dur": 1029.396,
    "args": {
      "External id": 2487612,"Record function id": 0, "Ev Idx": 4411
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.8)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662683.365, "dur": 348.979,
    "args": {
      "External id": 2487613,"Record function id": 0, "Ev Idx": 4412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662768.111, "dur": 4.755,
    "args": {
      "External id": 2487614,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662776.223, "dur": 1.435,
    "args": {
      "External id": 2487615,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662779.478, "dur": 1.346,
    "args": {
      "External id": 2487616,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662782.851, "dur": 1.156,
    "args": {
      "External id": 2487617,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662785.447, "dur": 1.064,
    "args": {
      "External id": 2487618,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662788.311, "dur": 2.476,
    "args": {
      "External id": 2487619,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662792.350, "dur": 1.217,
    "args": {
      "External id": 2487620,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662795.473, "dur": 1.113,
    "args": {
      "External id": 2487621,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662797.997, "dur": 1.172,
    "args": {
      "External id": 2487622,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662801.023, "dur": 0.771,
    "args": {
      "External id": 2487623,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662820.749, "dur": 144.293,
    "args": {
      "External id": 2487624,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662836.569, "dur": 123.917,
    "args": {
      "External id": 2487625,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662854.926, "dur": 12.125,
    "args": {
      "External id": 2487626,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662869.963, "dur": 63.708,
    "args": {
      "External id": 2487627,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662872.395, "dur": 60.883,
    "args": {
      "External id": 2487628,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662875.913, "dur": 5.768,
    "args": {
      "External id": 2487629,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657662883.180, "dur": 49.407,
    "args": {
      "External id": 2487630,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4429
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.7", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663126.964, "dur": 563.306,
    "args": {
      "External id": 2487631,"Record function id": 0, "Ev Idx": 4430
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.7)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663143.947, "dur": 534.234,
    "args": {
      "External id": 2487632,"Record function id": 0, "Ev Idx": 4431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663209.104, "dur": 5.666,
    "args": {
      "External id": 2487633,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663233.748, "dur": 31.232,
    "args": {
      "External id": 2487634,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663238.197, "dur": 3.521,
    "args": {
      "External id": 2487635,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663243.806, "dur": 0.611,
    "args": {
      "External id": 2487636,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663245.846, "dur": 0.342,
    "args": {
      "External id": 2487637,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663247.689, "dur": 0.441,
    "args": {
      "External id": 2487638,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663249.242, "dur": 0.408,
    "args": {
      "External id": 2487639,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663250.903, "dur": 0.525,
    "args": {
      "External id": 2487640,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663252.755, "dur": 0.704,
    "args": {
      "External id": 2487641,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663254.868, "dur": 0.545,
    "args": {
      "External id": 2487642,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663257.023, "dur": 2.934,
    "args": {
      "External id": 2487643,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663276.048, "dur": 30.745,
    "args": {
      "External id": 2487644,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663337.305, "dur": 95.334,
    "args": {
      "External id": 2487645,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663347.424, "dur": 3.148,
    "args": {
      "External id": 2487646,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663355.023, "dur": 10.487,
    "args": {
      "External id": 2487647,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663359.496, "dur": 5.600,
    "args": {
      "External id": 2487648,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663363.010, "dur": 0.770,
    "args": {
      "External id": 2487649,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663372.119, "dur": 23.771,
    "args": {
      "External id": 2487650,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663374.520, "dur": 0.517,
    "args": {
      "External id": 2487651,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663376.841, "dur": 0.507,
    "args": {
      "External id": 2487652,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663378.582, "dur": 0.386,
    "args": {
      "External id": 2487653,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663380.328, "dur": 0.537,
    "args": {
      "External id": 2487654,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663382.203, "dur": 0.450,
    "args": {
      "External id": 2487655,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663384.245, "dur": 2.740,
    "args": {
      "External id": 2487656,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663388.405, "dur": 0.490,
    "args": {
      "External id": 2487657,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663390.374, "dur": 0.754,
    "args": {
      "External id": 2487658,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663392.712, "dur": 0.607,
    "args": {
      "External id": 2487659,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663408.076, "dur": 17.124,
    "args": {
      "External id": 2487660,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663474.946, "dur": 133.589,
    "args": {
      "External id": 2487661,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663502.191, "dur": 102.580,
    "args": {
      "External id": 2487662,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4461, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663511.882, "dur": 88.280,
    "args": {
      "External id": 2487663,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663625.663, "dur": 2.148,
    "args": {
      "External id": 2487664,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4463, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663705.667, "dur": 1675.153,
    "args": {
      "External id": 2487665,"Sequence number": 24740438, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4464
    }
  },
  {
    "ph": "f", "id": 210, "pid": 1336757, "tid": 1381170, "ts": 1514657663705.667,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663816.002, "dur": 103.151,
    "args": {
      "External id": 2487666,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657663960.961, "dur": 83.818,
    "args": {
      "External id": 2487667,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664065.262, "dur": 56.730,
    "args": {
      "External id": 2487668,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664131.865, "dur": 32.564,
    "args": {
      "External id": 2487669,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664171.005, "dur": 45.915,
    "args": {
      "External id": 2487670,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664225.874, "dur": 28.054,
    "args": {
      "External id": 2487671,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664262.063, "dur": 42.301,
    "args": {
      "External id": 2487672,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664331.707, "dur": 25.056,
    "args": {
      "External id": 2487673,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664375.687, "dur": 29.078,
    "args": {
      "External id": 2487674,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664427.553, "dur": 20.454,
    "args": {
      "External id": 2487675,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664463.661, "dur": 15.995,
    "args": {
      "External id": 2487676,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664488.576, "dur": 28.936,
    "args": {
      "External id": 2487677,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664520.846, "dur": 51.350,
    "args": {
      "External id": 2487678,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664603.453, "dur": 178.674,
    "args": {
      "External id": 2487679,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664683.277, "dur": 8.775,
    "args": {
      "External id": 2487680,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664694.123, "dur": 2.606,
    "args": {
      "External id": 2487681,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664816.700, "dur": 24.575,
    "args": {
      "External id": 2487682,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664852.278, "dur": 15.409,
    "args": {
      "External id": 2487683,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664875.861, "dur": 42.830,
    "args": {
      "External id": 2487684,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664925.131, "dur": 36.964,
    "args": {
      "External id": 2487685,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657664969.076, "dur": 58.973,
    "args": {
      "External id": 2487686,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665038.651, "dur": 36.328,
    "args": {
      "External id": 2487687,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665081.688, "dur": 21.760,
    "args": {
      "External id": 2487688,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665121.336, "dur": 43.729,
    "args": {
      "External id": 2487689,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665186.389, "dur": 24.208,
    "args": {
      "External id": 2487690,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665227.965, "dur": 28.732,
    "args": {
      "External id": 2487691,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665271.327, "dur": 17.708,
    "args": {
      "External id": 2487692,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665306.014, "dur": 16.018,
    "args": {
      "External id": 2487693,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665335.888, "dur": 17.396,
    "args": {
      "External id": 2487694,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665424.146, "dur": 19.001,
    "args": {
      "External id": 2487695,"Record function id": 0, "Ev Idx": 4494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665427.516, "dur": 14.687,
    "args": {
      "External id": 2487696,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665435.201, "dur": 6.182,
    "args": {
      "External id": 2487697,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665437.120, "dur": 4.148,
    "args": {
      "External id": 2487698,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665447.000, "dur": 5.818,
    "args": {
      "External id": 2487699,"Record function id": 0, "Ev Idx": 4498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665448.474, "dur": 3.891,
    "args": {
      "External id": 2487700,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665449.520, "dur": 2.277,
    "args": {
      "External id": 2487701,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665450.420, "dur": 1.259,
    "args": {
      "External id": 2487702,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665456.152, "dur": 4.580,
    "args": {
      "External id": 2487703,"Record function id": 0, "Ev Idx": 4502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665457.583, "dur": 2.743,
    "args": {
      "External id": 2487704,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665458.213, "dur": 1.651,
    "args": {
      "External id": 2487705,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665458.847, "dur": 0.937,
    "args": {
      "External id": 2487706,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665463.953, "dur": 4.361,
    "args": {
      "External id": 2487707,"Record function id": 0, "Ev Idx": 4506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665465.112, "dur": 2.775,
    "args": {
      "External id": 2487708,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665465.742, "dur": 1.593,
    "args": {
      "External id": 2487709,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665466.394, "dur": 0.879,
    "args": {
      "External id": 2487710,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665471.391, "dur": 6.139,
    "args": {
      "External id": 2487711,"Record function id": 0, "Ev Idx": 4510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665472.327, "dur": 4.773,
    "args": {
      "External id": 2487712,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665473.171, "dur": 3.386,
    "args": {
      "External id": 2487713,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665473.495, "dur": 2.993,
    "args": {
      "External id": 2487714,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665480.674, "dur": 4.902,
    "args": {
      "External id": 2487715,"Record function id": 0, "Ev Idx": 4514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665481.995, "dur": 3.074,
    "args": {
      "External id": 2487716,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665482.688, "dur": 1.857,
    "args": {
      "External id": 2487717,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665483.199, "dur": 1.235,
    "args": {
      "External id": 2487718,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665488.854, "dur": 6.706,
    "args": {
      "External id": 2487719,"Record function id": 0, "Ev Idx": 4518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665490.112, "dur": 5.039,
    "args": {
      "External id": 2487720,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665490.565, "dur": 4.033,
    "args": {
      "External id": 2487721,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665490.845, "dur": 3.683,
    "args": {
      "External id": 2487722,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665498.607, "dur": 3.873,
    "args": {
      "External id": 2487723,"Record function id": 0, "Ev Idx": 4522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665499.671, "dur": 2.396,
    "args": {
      "External id": 2487724,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665500.146, "dur": 1.366,
    "args": {
      "External id": 2487725,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665500.554, "dur": 0.859,
    "args": {
      "External id": 2487726,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665505.517, "dur": 4.460,
    "args": {
      "External id": 2487727,"Record function id": 0, "Ev Idx": 4526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665506.767, "dur": 2.807,
    "args": {
      "External id": 2487728,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665507.294, "dur": 1.690,
    "args": {
      "External id": 2487729,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665507.969, "dur": 0.890,
    "args": {
      "External id": 2487730,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665513.774, "dur": 37030.247,
    "args": {
      "External id": 2487731,"Record function id": 0, "Sequence number": 24740437, "Fwd thread id": 1, "Ev Idx": 4530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665514.934, "dur": 37005.621,
    "args": {
      "External id": 2487732,"Sequence number": 24740437, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4531
    }
  },
  {
    "ph": "f", "id": 211, "pid": 1336757, "tid": 1381170, "ts": 1514657665514.934,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.8)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665562.437, "dur": 35.707,
    "args": {
      "External id": 2487733,"Record function id": 0, "Ev Idx": 4532
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.8)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665606.151, "dur": 67.319,
    "args": {
      "External id": 2487734,"Record function id": 0, "Ev Idx": 4533
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.8)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665682.538, "dur": 36830.029,
    "args": {
      "External id": 2487735,"Record function id": 0, "Ev Idx": 4534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665771.531, "dur": 7.350,
    "args": {
      "External id": 2487736,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665789.120, "dur": 5.213,
    "args": {
      "External id": 2487737,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665811.878, "dur": 35855.295,
    "args": {
      "External id": 2487738,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665825.395, "dur": 35832.865,
    "args": {
      "External id": 2487739,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665879.356, "dur": 13.773,
    "args": {
      "External id": 2487740,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665899.402, "dur": 35719.233,
    "args": {
      "External id": 2487741,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665902.158, "dur": 35715.721,
    "args": {
      "External id": 2487742,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665906.147, "dur": 7.340,
    "args": {
      "External id": 2487743,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657665915.253, "dur": 35699.263,
    "args": {
      "External id": 2487744,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657701753.573, "dur": 10.004,
    "args": {
      "External id": 2487745,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657701757.446, "dur": 5.790,
    "args": {
      "External id": 2487746,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657701791.758, "dur": 428.812,
    "args": {
      "External id": 2487747,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657701819.782, "dur": 395.173,
    "args": {
      "External id": 2487748,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4547, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657701830.747, "dur": 377.714,
    "args": {
      "External id": 2487749,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702244.216, "dur": 2.967,
    "args": {
      "External id": 2487750,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4549, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702308.331, "dur": 7.175,
    "args": {
      "External id": 2487751,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702360.254, "dur": 1.548,
    "args": {
      "External id": 2487752,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702377.351, "dur": 1.504,
    "args": {
      "External id": 2487753,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702389.840, "dur": 2.949,
    "args": {
      "External id": 2487754,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702407.547, "dur": 0.918,
    "args": {
      "External id": 2487755,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702418.408, "dur": 0.952,
    "args": {
      "External id": 2487756,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702430.413, "dur": 0.969,
    "args": {
      "External id": 2487757,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702442.080, "dur": 2.619,
    "args": {
      "External id": 2487758,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702454.333, "dur": 0.936,
    "args": {
      "External id": 2487759,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702560.976, "dur": 2826.298,
    "args": {
      "External id": 2487760,"Record function id": 0, "Ev Idx": 4559
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.7)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702582.930, "dur": 1046.613,
    "args": {
      "External id": 2487761,"Record function id": 0, "Ev Idx": 4560
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.7)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702597.592, "dur": 310.276,
    "args": {
      "External id": 2487762,"Record function id": 0, "Ev Idx": 4561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702681.294, "dur": 4.564,
    "args": {
      "External id": 2487763,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702688.931, "dur": 1.275,
    "args": {
      "External id": 2487764,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702692.887, "dur": 1.180,
    "args": {
      "External id": 2487765,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702695.686, "dur": 1.386,
    "args": {
      "External id": 2487766,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702698.478, "dur": 0.914,
    "args": {
      "External id": 2487767,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702700.776, "dur": 3.281,
    "args": {
      "External id": 2487768,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702706.432, "dur": 1.147,
    "args": {
      "External id": 2487769,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702708.954, "dur": 1.238,
    "args": {
      "External id": 2487770,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702711.623, "dur": 0.866,
    "args": {
      "External id": 2487771,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702713.838, "dur": 0.899,
    "args": {
      "External id": 2487772,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702733.794, "dur": 145.883,
    "args": {
      "External id": 2487773,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702749.487, "dur": 126.032,
    "args": {
      "External id": 2487774,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702765.954, "dur": 12.142,
    "args": {
      "External id": 2487775,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702781.082, "dur": 66.278,
    "args": {
      "External id": 2487776,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702783.452, "dur": 63.567,
    "args": {
      "External id": 2487777,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702787.753, "dur": 6.813,
    "args": {
      "External id": 2487778,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657702796.080, "dur": 50.479,
    "args": {
      "External id": 2487779,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4578
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.6", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703033.771, "dur": 588.021,
    "args": {
      "External id": 2487780,"Record function id": 0, "Ev Idx": 4579
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.6)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703049.769, "dur": 559.415,
    "args": {
      "External id": 2487781,"Record function id": 0, "Ev Idx": 4580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703114.110, "dur": 5.820,
    "args": {
      "External id": 2487782,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703139.611, "dur": 33.109,
    "args": {
      "External id": 2487783,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703144.779, "dur": 4.178,
    "args": {
      "External id": 2487784,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703151.319, "dur": 0.511,
    "args": {
      "External id": 2487785,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703153.530, "dur": 0.384,
    "args": {
      "External id": 2487786,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703156.564, "dur": 0.433,
    "args": {
      "External id": 2487787,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703158.244, "dur": 0.528,
    "args": {
      "External id": 2487788,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703160.165, "dur": 0.558,
    "args": {
      "External id": 2487789,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703162.691, "dur": 0.304,
    "args": {
      "External id": 2487790,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703164.377, "dur": 0.427,
    "args": {
      "External id": 2487791,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703166.259, "dur": 2.844,
    "args": {
      "External id": 2487792,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703183.026, "dur": 33.842,
    "args": {
      "External id": 2487793,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703246.162, "dur": 121.925,
    "args": {
      "External id": 2487794,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703255.948, "dur": 3.224,
    "args": {
      "External id": 2487795,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703264.178, "dur": 10.599,
    "args": {
      "External id": 2487796,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703268.665, "dur": 5.673,
    "args": {
      "External id": 2487797,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703272.416, "dur": 0.673,
    "args": {
      "External id": 2487798,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703281.652, "dur": 26.033,
    "args": {
      "External id": 2487799,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703283.611, "dur": 0.750,
    "args": {
      "External id": 2487800,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703286.017, "dur": 0.673,
    "args": {
      "External id": 2487801,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703288.003, "dur": 0.838,
    "args": {
      "External id": 2487802,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703290.364, "dur": 0.460,
    "args": {
      "External id": 2487803,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703292.261, "dur": 0.331,
    "args": {
      "External id": 2487804,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703294.747, "dur": 2.182,
    "args": {
      "External id": 2487805,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703298.338, "dur": 0.545,
    "args": {
      "External id": 2487806,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703300.163, "dur": 0.368,
    "args": {
      "External id": 2487807,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703301.865, "dur": 0.563,
    "args": {
      "External id": 2487808,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703340.108, "dur": 20.605,
    "args": {
      "External id": 2487809,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703410.150, "dur": 110.246,
    "args": {
      "External id": 2487810,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703435.313, "dur": 81.303,
    "args": {
      "External id": 2487811,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4610, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703444.491, "dur": 67.632,
    "args": {
      "External id": 2487812,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703553.573, "dur": 2.831,
    "args": {
      "External id": 2487813,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4612, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703637.092, "dur": 1730.779,
    "args": {
      "External id": 2487814,"Sequence number": 24740436, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4613
    }
  },
  {
    "ph": "f", "id": 212, "pid": 1336757, "tid": 1381170, "ts": 1514657703637.092,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703747.690, "dur": 105.951,
    "args": {
      "External id": 2487815,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703897.187, "dur": 42.152,
    "args": {
      "External id": 2487816,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657703954.866, "dur": 94.141,
    "args": {
      "External id": 2487817,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704063.353, "dur": 39.451,
    "args": {
      "External id": 2487818,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704114.736, "dur": 50.610,
    "args": {
      "External id": 2487819,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704174.646, "dur": 27.923,
    "args": {
      "External id": 2487820,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704210.374, "dur": 42.309,
    "args": {
      "External id": 2487821,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704281.303, "dur": 26.336,
    "args": {
      "External id": 2487822,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704331.012, "dur": 42.365,
    "args": {
      "External id": 2487823,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704398.828, "dur": 20.700,
    "args": {
      "External id": 2487824,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704435.004, "dur": 16.474,
    "args": {
      "External id": 2487825,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704460.055, "dur": 30.636,
    "args": {
      "External id": 2487826,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704494.153, "dur": 47.508,
    "args": {
      "External id": 2487827,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704577.471, "dur": 181.701,
    "args": {
      "External id": 2487828,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704659.056, "dur": 11.000,
    "args": {
      "External id": 2487829,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704671.869, "dur": 2.476,
    "args": {
      "External id": 2487830,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704796.354, "dur": 23.773,
    "args": {
      "External id": 2487831,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704831.738, "dur": 17.697,
    "args": {
      "External id": 2487832,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704857.048, "dur": 41.804,
    "args": {
      "External id": 2487833,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704904.030, "dur": 34.705,
    "args": {
      "External id": 2487834,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704944.529, "dur": 24.248,
    "args": {
      "External id": 2487835,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657704975.351, "dur": 68.383,
    "args": {
      "External id": 2487836,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705065.935, "dur": 33.474,
    "args": {
      "External id": 2487837,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705109.809, "dur": 32.924,
    "args": {
      "External id": 2487838,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705170.869, "dur": 24.368,
    "args": {
      "External id": 2487839,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705213.136, "dur": 27.289,
    "args": {
      "External id": 2487840,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705256.474, "dur": 17.168,
    "args": {
      "External id": 2487841,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705291.777, "dur": 15.067,
    "args": {
      "External id": 2487842,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705321.004, "dur": 20.226,
    "args": {
      "External id": 2487843,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705408.673, "dur": 14.794,
    "args": {
      "External id": 2487844,"Record function id": 0, "Ev Idx": 4643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705411.939, "dur": 10.641,
    "args": {
      "External id": 2487845,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705415.746, "dur": 5.743,
    "args": {
      "External id": 2487846,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705417.016, "dur": 4.378,
    "args": {
      "External id": 2487847,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705427.170, "dur": 4.870,
    "args": {
      "External id": 2487848,"Record function id": 0, "Ev Idx": 4647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705428.613, "dur": 3.012,
    "args": {
      "External id": 2487849,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705429.496, "dur": 1.654,
    "args": {
      "External id": 2487850,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705430.093, "dur": 0.990,
    "args": {
      "External id": 2487851,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705435.233, "dur": 3.932,
    "args": {
      "External id": 2487852,"Record function id": 0, "Ev Idx": 4651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705436.567, "dur": 2.164,
    "args": {
      "External id": 2487853,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705437.097, "dur": 1.207,
    "args": {
      "External id": 2487854,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705437.581, "dur": 0.655,
    "args": {
      "External id": 2487855,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705442.350, "dur": 4.608,
    "args": {
      "External id": 2487856,"Record function id": 0, "Ev Idx": 4655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705443.974, "dur": 2.553,
    "args": {
      "External id": 2487857,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705444.476, "dur": 1.495,
    "args": {
      "External id": 2487858,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705445.169, "dur": 0.681,
    "args": {
      "External id": 2487859,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705450.065, "dur": 6.000,
    "args": {
      "External id": 2487860,"Record function id": 0, "Ev Idx": 4659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705451.343, "dur": 4.313,
    "args": {
      "External id": 2487861,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705451.811, "dur": 3.411,
    "args": {
      "External id": 2487862,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705452.117, "dur": 3.016,
    "args": {
      "External id": 2487863,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705459.204, "dur": 4.539,
    "args": {
      "External id": 2487864,"Record function id": 0, "Ev Idx": 4663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705460.710, "dur": 2.600,
    "args": {
      "External id": 2487865,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705461.534, "dur": 1.357,
    "args": {
      "External id": 2487866,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705461.934, "dur": 0.856,
    "args": {
      "External id": 2487867,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705466.926, "dur": 4.081,
    "args": {
      "External id": 2487868,"Record function id": 0, "Ev Idx": 4667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705468.261, "dur": 2.323,
    "args": {
      "External id": 2487869,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705468.745, "dur": 1.126,
    "args": {
      "External id": 2487870,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705469.244, "dur": 0.559,
    "args": {
      "External id": 2487871,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705474.043, "dur": 3.526,
    "args": {
      "External id": 2487872,"Record function id": 0, "Ev Idx": 4671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705475.213, "dur": 1.931,
    "args": {
      "External id": 2487873,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705475.665, "dur": 1.083,
    "args": {
      "External id": 2487874,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705476.050, "dur": 0.614,
    "args": {
      "External id": 2487875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705480.628, "dur": 5.098,
    "args": {
      "External id": 2487876,"Record function id": 0, "Ev Idx": 4675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705481.777, "dur": 3.532,
    "args": {
      "External id": 2487877,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705482.243, "dur": 2.512,
    "args": {
      "External id": 2487878,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705483.978, "dur": 0.705,
    "args": {
      "External id": 2487879,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705489.287, "dur": 35927.706,
    "args": {
      "External id": 2487880,"Record function id": 0, "Sequence number": 24740435, "Fwd thread id": 1, "Ev Idx": 4679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705490.636, "dur": 35918.039,
    "args": {
      "External id": 2487881,"Sequence number": 24740435, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4680
    }
  },
  {
    "ph": "f", "id": 213, "pid": 1336757, "tid": 1381170, "ts": 1514657705490.636,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.7)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705518.200, "dur": 51.014,
    "args": {
      "External id": 2487882,"Record function id": 0, "Ev Idx": 4681
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.7)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705578.375, "dur": 68.885,
    "args": {
      "External id": 2487883,"Record function id": 0, "Ev Idx": 4682
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.7)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705652.966, "dur": 35748.395,
    "args": {
      "External id": 2487884,"Record function id": 0, "Ev Idx": 4683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705746.188, "dur": 6.905,
    "args": {
      "External id": 2487885,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705763.470, "dur": 4.986,
    "args": {
      "External id": 2487886,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705782.096, "dur": 34912.307,
    "args": {
      "External id": 2487887,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705795.788, "dur": 34890.742,
    "args": {
      "External id": 2487888,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705842.447, "dur": 14.009,
    "args": {
      "External id": 2487889,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705862.736, "dur": 34785.130,
    "args": {
      "External id": 2487890,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705868.917, "dur": 34778.200,
    "args": {
      "External id": 2487891,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705872.931, "dur": 5.120,
    "args": {
      "External id": 2487892,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657705879.571, "dur": 34763.950,
    "args": {
      "External id": 2487893,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657740780.998, "dur": 8.488,
    "args": {
      "External id": 2487894,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657740783.657, "dur": 5.483,
    "args": {
      "External id": 2487895,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657740816.778, "dur": 300.161,
    "args": {
      "External id": 2487896,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657740845.594, "dur": 266.459,
    "args": {
      "External id": 2487897,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4696, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657740856.289, "dur": 250.291,
    "args": {
      "External id": 2487898,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741140.463, "dur": 2.251,
    "args": {
      "External id": 2487899,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4698, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741197.197, "dur": 8.561,
    "args": {
      "External id": 2487900,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741249.336, "dur": 1.491,
    "args": {
      "External id": 2487901,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741266.787, "dur": 1.454,
    "args": {
      "External id": 2487902,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741280.854, "dur": 0.961,
    "args": {
      "External id": 2487903,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741292.475, "dur": 2.976,
    "args": {
      "External id": 2487904,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741305.599, "dur": 1.034,
    "args": {
      "External id": 2487905,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741317.967, "dur": 0.968,
    "args": {
      "External id": 2487906,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741329.430, "dur": 0.826,
    "args": {
      "External id": 2487907,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741341.578, "dur": 2.828,
    "args": {
      "External id": 2487908,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741430.001, "dur": 2805.195,
    "args": {
      "External id": 2487909,"Record function id": 0, "Ev Idx": 4708
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.6)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741448.811, "dur": 1039.635,
    "args": {
      "External id": 2487910,"Record function id": 0, "Ev Idx": 4709
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.6)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741461.384, "dur": 332.707,
    "args": {
      "External id": 2487911,"Record function id": 0, "Ev Idx": 4710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741561.946, "dur": 4.885,
    "args": {
      "External id": 2487912,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741570.059, "dur": 1.490,
    "args": {
      "External id": 2487913,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741573.312, "dur": 0.988,
    "args": {
      "External id": 2487914,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741576.567, "dur": 0.964,
    "args": {
      "External id": 2487915,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741579.237, "dur": 1.304,
    "args": {
      "External id": 2487916,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741583.191, "dur": 0.940,
    "args": {
      "External id": 2487917,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741585.936, "dur": 1.178,
    "args": {
      "External id": 2487918,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741588.738, "dur": 2.906,
    "args": {
      "External id": 2487919,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741593.250, "dur": 1.001,
    "args": {
      "External id": 2487920,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741596.783, "dur": 1.223,
    "args": {
      "External id": 2487921,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741616.325, "dur": 147.883,
    "args": {
      "External id": 2487922,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741632.016, "dur": 127.930,
    "args": {
      "External id": 2487923,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741648.906, "dur": 12.874,
    "args": {
      "External id": 2487924,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741664.594, "dur": 66.032,
    "args": {
      "External id": 2487925,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741667.085, "dur": 63.270,
    "args": {
      "External id": 2487926,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741674.045, "dur": 5.915,
    "args": {
      "External id": 2487927,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741681.540, "dur": 48.282,
    "args": {
      "External id": 2487928,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4727
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741885.385, "dur": 595.255,
    "args": {
      "External id": 2487929,"Record function id": 0, "Ev Idx": 4728
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.5)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741901.754, "dur": 567.105,
    "args": {
      "External id": 2487930,"Record function id": 0, "Ev Idx": 4729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657741968.213, "dur": 4.440,
    "args": {
      "External id": 2487931,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742028.248, "dur": 30.911,
    "args": {
      "External id": 2487932,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742033.000, "dur": 1.900,
    "args": {
      "External id": 2487933,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742037.396, "dur": 0.694,
    "args": {
      "External id": 2487934,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742040.471, "dur": 2.395,
    "args": {
      "External id": 2487935,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742044.175, "dur": 0.627,
    "args": {
      "External id": 2487936,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742045.965, "dur": 2.105,
    "args": {
      "External id": 2487937,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742049.384, "dur": 0.379,
    "args": {
      "External id": 2487938,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742050.999, "dur": 0.481,
    "args": {
      "External id": 2487939,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742053.673, "dur": 0.557,
    "args": {
      "External id": 2487940,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742055.704, "dur": 0.473,
    "args": {
      "External id": 2487941,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742070.864, "dur": 32.847,
    "args": {
      "External id": 2487942,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742134.317, "dur": 112.063,
    "args": {
      "External id": 2487943,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742144.730, "dur": 4.261,
    "args": {
      "External id": 2487944,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742153.582, "dur": 11.971,
    "args": {
      "External id": 2487945,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742157.617, "dur": 7.407,
    "args": {
      "External id": 2487946,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742161.300, "dur": 2.533,
    "args": {
      "External id": 2487947,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742172.210, "dur": 30.306,
    "args": {
      "External id": 2487948,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742174.386, "dur": 0.422,
    "args": {
      "External id": 2487949,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742176.650, "dur": 0.360,
    "args": {
      "External id": 2487950,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742182.048, "dur": 0.273,
    "args": {
      "External id": 2487951,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742183.576, "dur": 0.486,
    "args": {
      "External id": 2487952,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742185.321, "dur": 0.562,
    "args": {
      "External id": 2487953,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742187.075, "dur": 0.428,
    "args": {
      "External id": 2487954,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742188.952, "dur": 0.707,
    "args": {
      "External id": 2487955,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742190.823, "dur": 2.415,
    "args": {
      "External id": 2487956,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742194.445, "dur": 0.380,
    "args": {
      "External id": 2487957,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742219.294, "dur": 19.573,
    "args": {
      "External id": 2487958,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742292.426, "dur": 109.284,
    "args": {
      "External id": 2487959,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742319.631, "dur": 79.013,
    "args": {
      "External id": 2487960,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4759, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742329.214, "dur": 64.960,
    "args": {
      "External id": 2487961,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742416.396, "dur": 2.026,
    "args": {
      "External id": 2487962,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4761, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742495.289, "dur": 1718.895,
    "args": {
      "External id": 2487963,"Sequence number": 24740434, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4762
    }
  },
  {
    "ph": "f", "id": 214, "pid": 1336757, "tid": 1381170, "ts": 1514657742495.289,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742636.798, "dur": 111.106,
    "args": {
      "External id": 2487964,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742787.810, "dur": 44.068,
    "args": {
      "External id": 2487965,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742849.692, "dur": 52.552,
    "args": {
      "External id": 2487966,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742912.296, "dur": 32.372,
    "args": {
      "External id": 2487967,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657742951.490, "dur": 85.317,
    "args": {
      "External id": 2487968,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743047.793, "dur": 32.442,
    "args": {
      "External id": 2487969,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743090.457, "dur": 45.769,
    "args": {
      "External id": 2487970,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743165.394, "dur": 26.580,
    "args": {
      "External id": 2487971,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743214.984, "dur": 31.190,
    "args": {
      "External id": 2487972,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743266.842, "dur": 19.369,
    "args": {
      "External id": 2487973,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743302.025, "dur": 16.500,
    "args": {
      "External id": 2487974,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743328.625, "dur": 28.323,
    "args": {
      "External id": 2487975,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743360.084, "dur": 32.902,
    "args": {
      "External id": 2487976,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743425.853, "dur": 192.052,
    "args": {
      "External id": 2487977,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743503.035, "dur": 6.424,
    "args": {
      "External id": 2487978,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743511.142, "dur": 2.272,
    "args": {
      "External id": 2487979,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743654.672, "dur": 25.926,
    "args": {
      "External id": 2487980,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743692.240, "dur": 15.473,
    "args": {
      "External id": 2487981,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743715.840, "dur": 37.881,
    "args": {
      "External id": 2487982,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743761.722, "dur": 34.972,
    "args": {
      "External id": 2487983,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743802.765, "dur": 21.425,
    "args": {
      "External id": 2487984,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743828.722, "dur": 29.411,
    "args": {
      "External id": 2487985,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743863.716, "dur": 33.766,
    "args": {
      "External id": 2487986,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743912.617, "dur": 37.854,
    "args": {
      "External id": 2487987,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657743970.931, "dur": 57.545,
    "args": {
      "External id": 2487988,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744055.610, "dur": 27.243,
    "args": {
      "External id": 2487989,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744100.915, "dur": 17.314,
    "args": {
      "External id": 2487990,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744135.575, "dur": 18.953,
    "args": {
      "External id": 2487991,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744166.362, "dur": 17.197,
    "args": {
      "External id": 2487992,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744257.404, "dur": 18.382,
    "args": {
      "External id": 2487993,"Record function id": 0, "Ev Idx": 4792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744260.886, "dur": 13.948,
    "args": {
      "External id": 2487994,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744268.605, "dur": 5.290,
    "args": {
      "External id": 2487995,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744269.876, "dur": 3.918,
    "args": {
      "External id": 2487996,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744279.403, "dur": 6.804,
    "args": {
      "External id": 2487997,"Record function id": 0, "Ev Idx": 4796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744280.790, "dur": 4.861,
    "args": {
      "External id": 2487998,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744281.972, "dur": 3.140,
    "args": {
      "External id": 2487999,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744282.434, "dur": 2.582,
    "args": {
      "External id": 2488000,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744289.504, "dur": 4.204,
    "args": {
      "External id": 2488001,"Record function id": 0, "Ev Idx": 4800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744290.654, "dur": 2.625,
    "args": {
      "External id": 2488002,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744291.561, "dur": 1.317,
    "args": {
      "External id": 2488003,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744292.092, "dur": 0.721,
    "args": {
      "External id": 2488004,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744296.803, "dur": 4.206,
    "args": {
      "External id": 2488005,"Record function id": 0, "Ev Idx": 4804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744297.971, "dur": 2.530,
    "args": {
      "External id": 2488006,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744298.898, "dur": 1.173,
    "args": {
      "External id": 2488007,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744299.231, "dur": 0.776,
    "args": {
      "External id": 2488008,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744304.057, "dur": 3.669,
    "args": {
      "External id": 2488009,"Record function id": 0, "Ev Idx": 4808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744305.217, "dur": 2.081,
    "args": {
      "External id": 2488010,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744305.892, "dur": 0.992,
    "args": {
      "External id": 2488011,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744306.258, "dur": 0.560,
    "args": {
      "External id": 2488012,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744310.875, "dur": 5.132,
    "args": {
      "External id": 2488013,"Record function id": 0, "Ev Idx": 4812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744311.990, "dur": 3.564,
    "args": {
      "External id": 2488014,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744312.845, "dur": 2.316,
    "args": {
      "External id": 2488015,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744314.235, "dur": 0.817,
    "args": {
      "External id": 2488016,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744319.334, "dur": 7.084,
    "args": {
      "External id": 2488017,"Record function id": 0, "Ev Idx": 4816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744320.337, "dur": 5.659,
    "args": {
      "External id": 2488018,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744321.124, "dur": 4.446,
    "args": {
      "External id": 2488019,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744321.414, "dur": 0.549,
    "args": {
      "External id": 2488020,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744329.521, "dur": 3.681,
    "args": {
      "External id": 2488021,"Record function id": 0, "Ev Idx": 4820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744330.522, "dur": 2.254,
    "args": {
      "External id": 2488022,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744331.315, "dur": 1.066,
    "args": {
      "External id": 2488023,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744331.620, "dur": 0.696,
    "args": {
      "External id": 2488024,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744336.177, "dur": 6.537,
    "args": {
      "External id": 2488025,"Record function id": 0, "Ev Idx": 4824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744337.302, "dur": 5.012,
    "args": {
      "External id": 2488026,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744338.112, "dur": 3.803,
    "args": {
      "External id": 2488027,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744338.736, "dur": 3.111,
    "args": {
      "External id": 2488028,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744346.265, "dur": 36351.720,
    "args": {
      "External id": 2488029,"Record function id": 0, "Sequence number": 24740433, "Fwd thread id": 1, "Ev Idx": 4828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744347.707, "dur": 36341.232,
    "args": {
      "External id": 2488030,"Sequence number": 24740433, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4829
    }
  },
  {
    "ph": "f", "id": 215, "pid": 1336757, "tid": 1381170, "ts": 1514657744347.707,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.6)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744376.316, "dur": 33.064,
    "args": {
      "External id": 2488031,"Record function id": 0, "Ev Idx": 4830
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.6)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744415.978, "dur": 65.479,
    "args": {
      "External id": 2488032,"Record function id": 0, "Ev Idx": 4831
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.6)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744486.614, "dur": 36194.775,
    "args": {
      "External id": 2488033,"Record function id": 0, "Ev Idx": 4832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744592.428, "dur": 7.582,
    "args": {
      "External id": 2488034,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744612.889, "dur": 4.954,
    "args": {
      "External id": 2488035,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744632.538, "dur": 35246.698,
    "args": {
      "External id": 2488036,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744645.497, "dur": 35225.488,
    "args": {
      "External id": 2488037,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744690.170, "dur": 13.588,
    "args": {
      "External id": 2488038,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744709.934, "dur": 35122.980,
    "args": {
      "External id": 2488039,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744712.493, "dur": 35119.715,
    "args": {
      "External id": 2488040,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744716.589, "dur": 5.521,
    "args": {
      "External id": 2488041,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657744723.777, "dur": 35104.561,
    "args": {
      "External id": 2488042,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657779964.310, "dur": 8.978,
    "args": {
      "External id": 2488043,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657779967.055, "dur": 5.919,
    "args": {
      "External id": 2488044,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780026.438, "dur": 348.884,
    "args": {
      "External id": 2488045,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780054.674, "dur": 315.639,
    "args": {
      "External id": 2488046,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4845, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780066.145, "dur": 298.461,
    "args": {
      "External id": 2488047,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780397.293, "dur": 2.464,
    "args": {
      "External id": 2488048,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4847, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780455.341, "dur": 6.549,
    "args": {
      "External id": 2488049,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780504.405, "dur": 1.476,
    "args": {
      "External id": 2488050,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780522.634, "dur": 1.463,
    "args": {
      "External id": 2488051,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780554.733, "dur": 1.289,
    "args": {
      "External id": 2488052,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780568.710, "dur": 0.801,
    "args": {
      "External id": 2488053,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780579.447, "dur": 0.953,
    "args": {
      "External id": 2488054,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780594.117, "dur": 1.249,
    "args": {
      "External id": 2488055,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780609.042, "dur": 1.239,
    "args": {
      "External id": 2488056,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780620.010, "dur": 0.752,
    "args": {
      "External id": 2488057,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780712.772, "dur": 2789.551,
    "args": {
      "External id": 2488058,"Record function id": 0, "Ev Idx": 4857
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.5)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780732.226, "dur": 1037.570,
    "args": {
      "External id": 2488059,"Record function id": 0, "Ev Idx": 4858
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.5)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780744.641, "dur": 358.984,
    "args": {
      "External id": 2488060,"Record function id": 0, "Ev Idx": 4859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780833.413, "dur": 6.999,
    "args": {
      "External id": 2488061,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780843.515, "dur": 1.050,
    "args": {
      "External id": 2488062,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780846.226, "dur": 2.660,
    "args": {
      "External id": 2488063,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780852.182, "dur": 0.969,
    "args": {
      "External id": 2488064,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780854.593, "dur": 1.010,
    "args": {
      "External id": 2488065,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780857.133, "dur": 1.207,
    "args": {
      "External id": 2488066,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 4865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780860.064, "dur": 0.939,
    "args": {
      "External id": 2488067,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 4866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780864.395, "dur": 0.885,
    "args": {
      "External id": 2488068,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780867.104, "dur": 1.138,
    "args": {
      "External id": 2488069,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780869.671, "dur": 1.354,
    "args": {
      "External id": 2488070,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 4869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780887.684, "dur": 183.730,
    "args": {
      "External id": 2488071,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780903.397, "dur": 163.146,
    "args": {
      "External id": 2488072,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 4871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780921.174, "dur": 12.973,
    "args": {
      "External id": 2488073,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780937.265, "dur": 100.126,
    "args": {
      "External id": 2488074,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 4873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780939.584, "dur": 97.324,
    "args": {
      "External id": 2488075,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 4874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780943.648, "dur": 6.071,
    "args": {
      "External id": 2488076,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657780951.386, "dur": 84.525,
    "args": {
      "External id": 2488077,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 4876
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781199.921, "dur": 561.862,
    "args": {
      "External id": 2488078,"Record function id": 0, "Ev Idx": 4877
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.4)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781219.298, "dur": 526.944,
    "args": {
      "External id": 2488079,"Record function id": 0, "Ev Idx": 4878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781282.994, "dur": 6.088,
    "args": {
      "External id": 2488080,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781308.415, "dur": 27.285,
    "args": {
      "External id": 2488081,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781313.005, "dur": 1.623,
    "args": {
      "External id": 2488082,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781316.864, "dur": 0.557,
    "args": {
      "External id": 2488083,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781318.793, "dur": 0.362,
    "args": {
      "External id": 2488084,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781320.785, "dur": 0.449,
    "args": {
      "External id": 2488085,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781322.537, "dur": 0.290,
    "args": {
      "External id": 2488086,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781324.032, "dur": 2.516,
    "args": {
      "External id": 2488087,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781328.088, "dur": 0.269,
    "args": {
      "External id": 2488088,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781330.003, "dur": 0.342,
    "args": {
      "External id": 2488089,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781332.402, "dur": 0.508,
    "args": {
      "External id": 2488090,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781346.627, "dur": 29.539,
    "args": {
      "External id": 2488091,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781406.371, "dur": 97.587,
    "args": {
      "External id": 2488092,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 4891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781415.608, "dur": 2.934,
    "args": {
      "External id": 2488093,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781423.263, "dur": 9.891,
    "args": {
      "External id": 2488094,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781427.196, "dur": 5.515,
    "args": {
      "External id": 2488095,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 4894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781431.018, "dur": 0.545,
    "args": {
      "External id": 2488096,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 4895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781439.905, "dur": 27.803,
    "args": {
      "External id": 2488097,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 4896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781442.694, "dur": 0.508,
    "args": {
      "External id": 2488098,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781445.099, "dur": 0.496,
    "args": {
      "External id": 2488099,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781447.386, "dur": 2.282,
    "args": {
      "External id": 2488100,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781451.689, "dur": 0.500,
    "args": {
      "External id": 2488101,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781453.714, "dur": 0.261,
    "args": {
      "External id": 2488102,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781455.462, "dur": 0.260,
    "args": {
      "External id": 2488103,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781456.983, "dur": 0.398,
    "args": {
      "External id": 2488104,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781459.061, "dur": 0.387,
    "args": {
      "External id": 2488105,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781460.803, "dur": 0.481,
    "args": {
      "External id": 2488106,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781478.194, "dur": 18.903,
    "args": {
      "External id": 2488107,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 4906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781564.191, "dur": 118.344,
    "args": {
      "External id": 2488108,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 4907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781590.633, "dur": 88.525,
    "args": {
      "External id": 2488109,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4908, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781603.888, "dur": 71.229,
    "args": {
      "External id": 2488110,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 4909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781697.327, "dur": 1.979,
    "args": {
      "External id": 2488111,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4910, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781777.039, "dur": 1705.084,
    "args": {
      "External id": 2488112,"Sequence number": 24740432, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4911
    }
  },
  {
    "ph": "f", "id": 216, "pid": 1336757, "tid": 1381170, "ts": 1514657781777.039,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657781894.201, "dur": 148.036,
    "args": {
      "External id": 2488113,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 4912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782091.217, "dur": 47.458,
    "args": {
      "External id": 2488114,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 4913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782155.423, "dur": 58.596,
    "args": {
      "External id": 2488115,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 4914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782223.957, "dur": 33.508,
    "args": {
      "External id": 2488116,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782265.567, "dur": 46.425,
    "args": {
      "External id": 2488117,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782319.785, "dur": 27.313,
    "args": {
      "External id": 2488118,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 4917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782355.935, "dur": 43.494,
    "args": {
      "External id": 2488119,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 4918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782425.429, "dur": 24.123,
    "args": {
      "External id": 2488120,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 4919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782471.716, "dur": 31.687,
    "args": {
      "External id": 2488121,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782524.109, "dur": 35.378,
    "args": {
      "External id": 2488122,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782578.132, "dur": 18.525,
    "args": {
      "External id": 2488123,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782605.913, "dur": 33.257,
    "args": {
      "External id": 2488124,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782642.516, "dur": 32.557,
    "args": {
      "External id": 2488125,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782705.676, "dur": 175.149,
    "args": {
      "External id": 2488126,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 4925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782787.656, "dur": 6.896,
    "args": {
      "External id": 2488127,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782796.092, "dur": 3.253,
    "args": {
      "External id": 2488128,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782913.914, "dur": 23.965,
    "args": {
      "External id": 2488129,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782956.323, "dur": 15.088,
    "args": {
      "External id": 2488130,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657782978.902, "dur": 76.322,
    "args": {
      "External id": 2488131,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783064.009, "dur": 35.852,
    "args": {
      "External id": 2488132,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783109.820, "dur": 21.061,
    "args": {
      "External id": 2488133,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783134.950, "dur": 30.419,
    "args": {
      "External id": 2488134,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783170.717, "dur": 35.068,
    "args": {
      "External id": 2488135,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 4934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783217.805, "dur": 40.188,
    "args": {
      "External id": 2488136,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 4935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783284.565, "dur": 24.306,
    "args": {
      "External id": 2488137,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 4936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783327.161, "dur": 29.215,
    "args": {
      "External id": 2488138,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 4937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783371.395, "dur": 17.705,
    "args": {
      "External id": 2488139,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 4938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783407.839, "dur": 15.544,
    "args": {
      "External id": 2488140,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 4939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783435.791, "dur": 16.787,
    "args": {
      "External id": 2488141,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 4940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783540.853, "dur": 17.914,
    "args": {
      "External id": 2488142,"Record function id": 0, "Ev Idx": 4941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783546.019, "dur": 11.475,
    "args": {
      "External id": 2488143,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783550.167, "dur": 5.978,
    "args": {
      "External id": 2488144,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783551.621, "dur": 4.277,
    "args": {
      "External id": 2488145,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783564.405, "dur": 4.679,
    "args": {
      "External id": 2488146,"Record function id": 0, "Ev Idx": 4945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783566.017, "dur": 2.614,
    "args": {
      "External id": 2488147,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783566.679, "dur": 1.424,
    "args": {
      "External id": 2488148,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783567.237, "dur": 0.770,
    "args": {
      "External id": 2488149,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783572.254, "dur": 4.323,
    "args": {
      "External id": 2488150,"Record function id": 0, "Ev Idx": 4949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783573.443, "dur": 2.657,
    "args": {
      "External id": 2488151,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783573.977, "dur": 1.720,
    "args": {
      "External id": 2488152,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783574.706, "dur": 0.885,
    "args": {
      "External id": 2488153,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 4952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783579.752, "dur": 3.473,
    "args": {
      "External id": 2488154,"Record function id": 0, "Ev Idx": 4953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783580.812, "dur": 1.982,
    "args": {
      "External id": 2488155,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783581.405, "dur": 1.002,
    "args": {
      "External id": 2488156,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783581.701, "dur": 0.636,
    "args": {
      "External id": 2488157,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 4956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783586.336, "dur": 5.614,
    "args": {
      "External id": 2488158,"Record function id": 0, "Ev Idx": 4957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783587.323, "dur": 4.222,
    "args": {
      "External id": 2488159,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783588.057, "dur": 3.104,
    "args": {
      "External id": 2488160,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783588.415, "dur": 2.675,
    "args": {
      "External id": 2488161,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783595.063, "dur": 3.957,
    "args": {
      "External id": 2488162,"Record function id": 0, "Ev Idx": 4961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783596.018, "dur": 2.565,
    "args": {
      "External id": 2488163,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783596.504, "dur": 1.443,
    "args": {
      "External id": 2488164,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783596.912, "dur": 0.964,
    "args": {
      "External id": 2488165,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783602.248, "dur": 3.370,
    "args": {
      "External id": 2488166,"Record function id": 0, "Ev Idx": 4965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783603.283, "dur": 1.881,
    "args": {
      "External id": 2488167,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783603.731, "dur": 0.950,
    "args": {
      "External id": 2488168,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783604.009, "dur": 0.605,
    "args": {
      "External id": 2488169,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783608.607, "dur": 4.431,
    "args": {
      "External id": 2488170,"Record function id": 0, "Ev Idx": 4969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783609.529, "dur": 3.098,
    "args": {
      "External id": 2488171,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783610.176, "dur": 2.037,
    "args": {
      "External id": 2488172,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783611.606, "dur": 0.509,
    "args": {
      "External id": 2488173,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 4972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783616.064, "dur": 3.395,
    "args": {
      "External id": 2488174,"Record function id": 0, "Ev Idx": 4973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783616.967, "dur": 2.069,
    "args": {
      "External id": 2488175,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783617.489, "dur": 1.138,
    "args": {
      "External id": 2488176,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783617.878, "dur": 0.657,
    "args": {
      "External id": 2488177,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 4976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783623.190, "dur": 38585.903,
    "args": {
      "External id": 2488178,"Record function id": 0, "Sequence number": 24740431, "Fwd thread id": 1, "Ev Idx": 4977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783625.042, "dur": 38574.853,
    "args": {
      "External id": 2488179,"Sequence number": 24740431, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 4978
    }
  },
  {
    "ph": "f", "id": 217, "pid": 1336757, "tid": 1381170, "ts": 1514657783625.042,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.5)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783653.418, "dur": 39.685,
    "args": {
      "External id": 2488180,"Record function id": 0, "Ev Idx": 4979
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.5)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783700.478, "dur": 68.824,
    "args": {
      "External id": 2488181,"Record function id": 0, "Ev Idx": 4980
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.5)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783774.873, "dur": 38417.524,
    "args": {
      "External id": 2488182,"Record function id": 0, "Ev Idx": 4981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783869.104, "dur": 6.725,
    "args": {
      "External id": 2488183,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783885.382, "dur": 4.494,
    "args": {
      "External id": 2488184,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 4983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783908.992, "dur": 37413.220,
    "args": {
      "External id": 2488185,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783921.988, "dur": 37391.943,
    "args": {
      "External id": 2488186,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 4985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657783972.359, "dur": 49.655,
    "args": {
      "External id": 2488187,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657784029.252, "dur": 37245.702,
    "args": {
      "External id": 2488188,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 4987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657784031.708, "dur": 37242.620,
    "args": {
      "External id": 2488189,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 4988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657784036.382, "dur": 6.643,
    "args": {
      "External id": 2488190,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657784044.631, "dur": 37226.021,
    "args": {
      "External id": 2488191,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 4990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657821408.039, "dur": 8.611,
    "args": {
      "External id": 2488192,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 4991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657821410.708, "dur": 5.634,
    "args": {
      "External id": 2488193,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 4992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657821444.106, "dur": 406.091,
    "args": {
      "External id": 2488194,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 4993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657821473.851, "dur": 370.870,
    "args": {
      "External id": 2488195,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 4994, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657821485.147, "dur": 353.257,
    "args": {
      "External id": 2488196,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 4995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657821873.859, "dur": 2.390,
    "args": {
      "External id": 2488197,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 4996, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657821937.408, "dur": 8.338,
    "args": {
      "External id": 2488198,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822024.244, "dur": 2.290,
    "args": {
      "External id": 2488199,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822045.781, "dur": 1.198,
    "args": {
      "External id": 2488200,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 4999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822060.063, "dur": 0.893,
    "args": {
      "External id": 2488201,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822072.775, "dur": 2.799,
    "args": {
      "External id": 2488202,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822086.592, "dur": 0.916,
    "args": {
      "External id": 2488203,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822098.861, "dur": 0.952,
    "args": {
      "External id": 2488204,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822110.881, "dur": 1.156,
    "args": {
      "External id": 2488205,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822125.039, "dur": 2.976,
    "args": {
      "External id": 2488206,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822224.071, "dur": 2803.578,
    "args": {
      "External id": 2488207,"Record function id": 0, "Ev Idx": 5006
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.4)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822243.628, "dur": 1067.555,
    "args": {
      "External id": 2488208,"Record function id": 0, "Ev Idx": 5007
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.4)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822257.623, "dur": 345.954,
    "args": {
      "External id": 2488209,"Record function id": 0, "Ev Idx": 5008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822342.285, "dur": 3.928,
    "args": {
      "External id": 2488210,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822349.194, "dur": 1.062,
    "args": {
      "External id": 2488211,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822353.010, "dur": 1.006,
    "args": {
      "External id": 2488212,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822355.645, "dur": 1.004,
    "args": {
      "External id": 2488213,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822358.135, "dur": 0.896,
    "args": {
      "External id": 2488214,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822360.657, "dur": 0.844,
    "args": {
      "External id": 2488215,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822364.458, "dur": 0.749,
    "args": {
      "External id": 2488216,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822366.708, "dur": 2.713,
    "args": {
      "External id": 2488217,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822370.962, "dur": 1.084,
    "args": {
      "External id": 2488218,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822373.922, "dur": 1.324,
    "args": {
      "External id": 2488219,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822394.053, "dur": 176.513,
    "args": {
      "External id": 2488220,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822410.514, "dur": 154.707,
    "args": {
      "External id": 2488221,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822431.351, "dur": 12.577,
    "args": {
      "External id": 2488222,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822447.047, "dur": 68.908,
    "args": {
      "External id": 2488223,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822449.486, "dur": 66.130,
    "args": {
      "External id": 2488224,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822453.695, "dur": 5.935,
    "args": {
      "External id": 2488225,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822461.138, "dur": 53.924,
    "args": {
      "External id": 2488226,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5025
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822707.378, "dur": 596.378,
    "args": {
      "External id": 2488227,"Record function id": 0, "Ev Idx": 5026
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.3)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822723.410, "dur": 565.297,
    "args": {
      "External id": 2488228,"Record function id": 0, "Ev Idx": 5027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822787.127, "dur": 5.536,
    "args": {
      "External id": 2488229,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822812.138, "dur": 33.191,
    "args": {
      "External id": 2488230,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822819.806, "dur": 1.751,
    "args": {
      "External id": 2488231,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822823.917, "dur": 0.492,
    "args": {
      "External id": 2488232,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822825.820, "dur": 2.383,
    "args": {
      "External id": 2488233,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822830.706, "dur": 0.555,
    "args": {
      "External id": 2488234,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822833.053, "dur": 0.560,
    "args": {
      "External id": 2488235,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822835.103, "dur": 0.772,
    "args": {
      "External id": 2488236,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822837.644, "dur": 0.376,
    "args": {
      "External id": 2488237,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822839.667, "dur": 0.368,
    "args": {
      "External id": 2488238,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822842.213, "dur": 0.276,
    "args": {
      "External id": 2488239,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822856.867, "dur": 34.390,
    "args": {
      "External id": 2488240,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822920.578, "dur": 144.015,
    "args": {
      "External id": 2488241,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822930.048, "dur": 2.888,
    "args": {
      "External id": 2488242,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822937.617, "dur": 11.531,
    "args": {
      "External id": 2488243,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822941.331, "dur": 7.379,
    "args": {
      "External id": 2488244,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822944.902, "dur": 2.500,
    "args": {
      "External id": 2488245,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822955.866, "dur": 62.436,
    "args": {
      "External id": 2488246,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822958.262, "dur": 0.329,
    "args": {
      "External id": 2488247,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822960.251, "dur": 0.374,
    "args": {
      "External id": 2488248,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822962.840, "dur": 0.429,
    "args": {
      "External id": 2488249,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822965.048, "dur": 0.384,
    "args": {
      "External id": 2488250,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822966.811, "dur": 0.333,
    "args": {
      "External id": 2488251,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822968.538, "dur": 0.376,
    "args": {
      "External id": 2488252,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822970.287, "dur": 0.331,
    "args": {
      "External id": 2488253,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822972.118, "dur": 2.110,
    "args": {
      "External id": 2488254,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657822975.801, "dur": 0.271,
    "args": {
      "External id": 2488255,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657823034.249, "dur": 22.821,
    "args": {
      "External id": 2488256,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657823109.494, "dur": 114.736,
    "args": {
      "External id": 2488257,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657823136.590, "dur": 84.396,
    "args": {
      "External id": 2488258,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5057, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657823146.210, "dur": 70.792,
    "args": {
      "External id": 2488259,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657823239.534, "dur": 1.839,
    "args": {
      "External id": 2488260,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5059, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657823318.974, "dur": 1652.272,
    "args": {
      "External id": 2488261,"Sequence number": 24740430, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5060
    }
  },
  {
    "ph": "f", "id": 218, "pid": 1336757, "tid": 1381170, "ts": 1514657823318.974,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657823433.368, "dur": 127.636,
    "args": {
      "External id": 2488262,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657823605.966, "dur": 39.749,
    "args": {
      "External id": 2488263,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657823664.821, "dur": 53.689,
    "args": {
      "External id": 2488264,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657823728.054, "dur": 32.420,
    "args": {
      "External id": 2488265,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657823769.765, "dur": 45.195,
    "args": {
      "External id": 2488266,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657823821.595, "dur": 30.352,
    "args": {
      "External id": 2488267,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657823860.690, "dur": 43.268,
    "args": {
      "External id": 2488268,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657823925.384, "dur": 25.047,
    "args": {
      "External id": 2488269,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657823969.260, "dur": 66.826,
    "args": {
      "External id": 2488270,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824064.081, "dur": 19.800,
    "args": {
      "External id": 2488271,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824099.057, "dur": 19.274,
    "args": {
      "External id": 2488272,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824128.576, "dur": 34.103,
    "args": {
      "External id": 2488273,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824165.866, "dur": 34.184,
    "args": {
      "External id": 2488274,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824228.124, "dur": 174.713,
    "args": {
      "External id": 2488275,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824310.212, "dur": 6.107,
    "args": {
      "External id": 2488276,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824318.203, "dur": 2.826,
    "args": {
      "External id": 2488277,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824436.918, "dur": 23.545,
    "args": {
      "External id": 2488278,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824471.986, "dur": 16.095,
    "args": {
      "External id": 2488279,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824495.003, "dur": 48.864,
    "args": {
      "External id": 2488280,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824554.069, "dur": 39.835,
    "args": {
      "External id": 2488281,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824600.713, "dur": 34.060,
    "args": {
      "External id": 2488282,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824645.002, "dur": 37.637,
    "args": {
      "External id": 2488283,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824689.917, "dur": 21.517,
    "args": {
      "External id": 2488284,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824720.633, "dur": 32.626,
    "args": {
      "External id": 2488285,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824774.673, "dur": 26.738,
    "args": {
      "External id": 2488286,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824819.847, "dur": 25.399,
    "args": {
      "External id": 2488287,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824865.456, "dur": 16.791,
    "args": {
      "External id": 2488288,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824899.506, "dur": 15.191,
    "args": {
      "External id": 2488289,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657824927.187, "dur": 18.341,
    "args": {
      "External id": 2488290,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825050.060, "dur": 15.651,
    "args": {
      "External id": 2488291,"Record function id": 0, "Ev Idx": 5090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825053.586, "dur": 11.164,
    "args": {
      "External id": 2488292,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825057.991, "dur": 5.522,
    "args": {
      "External id": 2488293,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825059.338, "dur": 4.086,
    "args": {
      "External id": 2488294,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825069.635, "dur": 4.735,
    "args": {
      "External id": 2488295,"Record function id": 0, "Ev Idx": 5094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825071.057, "dur": 2.884,
    "args": {
      "External id": 2488296,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825071.735, "dur": 1.706,
    "args": {
      "External id": 2488297,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825072.292, "dur": 1.020,
    "args": {
      "External id": 2488298,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825080.655, "dur": 6.445,
    "args": {
      "External id": 2488299,"Record function id": 0, "Ev Idx": 5098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825081.932, "dur": 4.731,
    "args": {
      "External id": 2488300,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825082.595, "dur": 3.553,
    "args": {
      "External id": 2488301,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825083.138, "dur": 2.886,
    "args": {
      "External id": 2488302,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825090.227, "dur": 4.343,
    "args": {
      "External id": 2488303,"Record function id": 0, "Ev Idx": 5102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825091.547, "dur": 2.586,
    "args": {
      "External id": 2488304,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825092.179, "dur": 1.538,
    "args": {
      "External id": 2488305,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825092.602, "dur": 1.051,
    "args": {
      "External id": 2488306,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825097.667, "dur": 3.971,
    "args": {
      "External id": 2488307,"Record function id": 0, "Ev Idx": 5106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825098.877, "dur": 2.345,
    "args": {
      "External id": 2488308,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825099.419, "dur": 1.361,
    "args": {
      "External id": 2488309,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825099.880, "dur": 0.833,
    "args": {
      "External id": 2488310,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825104.745, "dur": 5.182,
    "args": {
      "External id": 2488311,"Record function id": 0, "Ev Idx": 5110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825105.750, "dur": 3.740,
    "args": {
      "External id": 2488312,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825106.270, "dur": 2.711,
    "args": {
      "External id": 2488313,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825108.299, "dur": 0.615,
    "args": {
      "External id": 2488314,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825113.175, "dur": 3.463,
    "args": {
      "External id": 2488315,"Record function id": 0, "Ev Idx": 5114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825114.304, "dur": 1.949,
    "args": {
      "External id": 2488316,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825114.806, "dur": 1.040,
    "args": {
      "External id": 2488317,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825115.128, "dur": 0.651,
    "args": {
      "External id": 2488318,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825119.714, "dur": 29.425,
    "args": {
      "External id": 2488319,"Record function id": 0, "Ev Idx": 5118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825120.771, "dur": 24.673,
    "args": {
      "External id": 2488320,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825143.854, "dur": 1.101,
    "args": {
      "External id": 2488321,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825144.278, "dur": 0.602,
    "args": {
      "External id": 2488322,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825152.321, "dur": 9.021,
    "args": {
      "External id": 2488323,"Record function id": 0, "Ev Idx": 5122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825157.670, "dur": 3.236,
    "args": {
      "External id": 2488324,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825158.119, "dur": 2.371,
    "args": {
      "External id": 2488325,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825159.750, "dur": 0.641,
    "args": {
      "External id": 2488326,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825165.717, "dur": 36402.910,
    "args": {
      "External id": 2488327,"Record function id": 0, "Sequence number": 24740429, "Fwd thread id": 1, "Ev Idx": 5126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825167.581, "dur": 36391.974,
    "args": {
      "External id": 2488328,"Sequence number": 24740429, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5127
    }
  },
  {
    "ph": "f", "id": 219, "pid": 1336757, "tid": 1381170, "ts": 1514657825167.581,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.4)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825197.323, "dur": 39.504,
    "args": {
      "External id": 2488329,"Record function id": 0, "Ev Idx": 5128
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.4)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825244.199, "dur": 68.715,
    "args": {
      "External id": 2488330,"Record function id": 0, "Ev Idx": 5129
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.4)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825318.942, "dur": 36232.196,
    "args": {
      "External id": 2488331,"Record function id": 0, "Ev Idx": 5130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825409.499, "dur": 7.190,
    "args": {
      "External id": 2488332,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825426.371, "dur": 6.174,
    "args": {
      "External id": 2488333,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825445.856, "dur": 35294.936,
    "args": {
      "External id": 2488334,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825458.610, "dur": 35273.723,
    "args": {
      "External id": 2488335,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825503.947, "dur": 13.563,
    "args": {
      "External id": 2488336,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825525.206, "dur": 35165.858,
    "args": {
      "External id": 2488337,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825543.697, "dur": 35146.591,
    "args": {
      "External id": 2488338,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825548.228, "dur": 6.283,
    "args": {
      "External id": 2488339,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657825556.356, "dur": 35130.564,
    "args": {
      "External id": 2488340,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657860824.834, "dur": 9.088,
    "args": {
      "External id": 2488341,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657860827.634, "dur": 5.950,
    "args": {
      "External id": 2488342,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657860862.925, "dur": 382.961,
    "args": {
      "External id": 2488343,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657860895.541, "dur": 344.843,
    "args": {
      "External id": 2488344,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5143, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657860906.330, "dur": 328.630,
    "args": {
      "External id": 2488345,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861271.647, "dur": 2.417,
    "args": {
      "External id": 2488346,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5145, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861329.676, "dur": 6.454,
    "args": {
      "External id": 2488347,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861379.762, "dur": 1.339,
    "args": {
      "External id": 2488348,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861396.436, "dur": 1.415,
    "args": {
      "External id": 2488349,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861409.975, "dur": 2.788,
    "args": {
      "External id": 2488350,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861424.009, "dur": 1.024,
    "args": {
      "External id": 2488351,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861434.909, "dur": 0.647,
    "args": {
      "External id": 2488352,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861446.834, "dur": 1.038,
    "args": {
      "External id": 2488353,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861458.486, "dur": 2.887,
    "args": {
      "External id": 2488354,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861475.475, "dur": 0.840,
    "args": {
      "External id": 2488355,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861586.888, "dur": 2750.673,
    "args": {
      "External id": 2488356,"Record function id": 0, "Ev Idx": 5155
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.3)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861606.721, "dur": 1038.921,
    "args": {
      "External id": 2488357,"Record function id": 0, "Ev Idx": 5156
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.3)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861620.715, "dur": 314.375,
    "args": {
      "External id": 2488358,"Record function id": 0, "Ev Idx": 5157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861704.580, "dur": 4.550,
    "args": {
      "External id": 2488359,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861715.605, "dur": 1.335,
    "args": {
      "External id": 2488360,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861719.639, "dur": 0.984,
    "args": {
      "External id": 2488361,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861722.350, "dur": 1.040,
    "args": {
      "External id": 2488362,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861725.116, "dur": 0.859,
    "args": {
      "External id": 2488363,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861727.520, "dur": 2.825,
    "args": {
      "External id": 2488364,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861733.129, "dur": 0.839,
    "args": {
      "External id": 2488365,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861735.414, "dur": 1.364,
    "args": {
      "External id": 2488366,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861738.356, "dur": 1.199,
    "args": {
      "External id": 2488367,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861740.944, "dur": 1.350,
    "args": {
      "External id": 2488368,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861759.887, "dur": 147.792,
    "args": {
      "External id": 2488369,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861778.504, "dur": 124.770,
    "args": {
      "External id": 2488370,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861794.001, "dur": 11.325,
    "args": {
      "External id": 2488371,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861808.632, "dur": 66.311,
    "args": {
      "External id": 2488372,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861811.512, "dur": 63.126,
    "args": {
      "External id": 2488373,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861815.750, "dur": 6.808,
    "args": {
      "External id": 2488374,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657861824.069, "dur": 50.153,
    "args": {
      "External id": 2488375,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5174
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862071.377, "dur": 566.388,
    "args": {
      "External id": 2488376,"Record function id": 0, "Ev Idx": 5175
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.2)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862087.285, "dur": 538.847,
    "args": {
      "External id": 2488377,"Record function id": 0, "Ev Idx": 5176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862150.672, "dur": 5.570,
    "args": {
      "External id": 2488378,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862178.854, "dur": 29.238,
    "args": {
      "External id": 2488379,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862183.446, "dur": 3.534,
    "args": {
      "External id": 2488380,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862188.950, "dur": 0.603,
    "args": {
      "External id": 2488381,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862190.940, "dur": 0.525,
    "args": {
      "External id": 2488382,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862193.058, "dur": 0.517,
    "args": {
      "External id": 2488383,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862195.124, "dur": 0.602,
    "args": {
      "External id": 2488384,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862196.861, "dur": 1.131,
    "args": {
      "External id": 2488385,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862199.254, "dur": 0.696,
    "args": {
      "External id": 2488386,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862200.820, "dur": 0.331,
    "args": {
      "External id": 2488387,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862203.098, "dur": 2.019,
    "args": {
      "External id": 2488388,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862218.840, "dur": 34.572,
    "args": {
      "External id": 2488389,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862283.210, "dur": 103.287,
    "args": {
      "External id": 2488390,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862297.066, "dur": 3.540,
    "args": {
      "External id": 2488391,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862305.489, "dur": 9.533,
    "args": {
      "External id": 2488392,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862309.545, "dur": 5.049,
    "args": {
      "External id": 2488393,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862312.741, "dur": 0.671,
    "args": {
      "External id": 2488394,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862321.405, "dur": 24.307,
    "args": {
      "External id": 2488395,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862323.652, "dur": 0.267,
    "args": {
      "External id": 2488396,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862325.108, "dur": 0.548,
    "args": {
      "External id": 2488397,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862327.207, "dur": 0.258,
    "args": {
      "External id": 2488398,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862328.461, "dur": 0.555,
    "args": {
      "External id": 2488399,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862330.392, "dur": 0.471,
    "args": {
      "External id": 2488400,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862332.202, "dur": 2.291,
    "args": {
      "External id": 2488401,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862335.757, "dur": 0.321,
    "args": {
      "External id": 2488402,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862337.321, "dur": 0.625,
    "args": {
      "External id": 2488403,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862339.194, "dur": 0.300,
    "args": {
      "External id": 2488404,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862360.465, "dur": 18.928,
    "args": {
      "External id": 2488405,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862427.061, "dur": 127.581,
    "args": {
      "External id": 2488406,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862451.340, "dur": 99.598,
    "args": {
      "External id": 2488407,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5206, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862460.463, "dur": 85.329,
    "args": {
      "External id": 2488408,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862569.677, "dur": 2.400,
    "args": {
      "External id": 2488409,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5208, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862652.483, "dur": 1663.990,
    "args": {
      "External id": 2488410,"Sequence number": 24740428, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5209
    }
  },
  {
    "ph": "f", "id": 220, "pid": 1336757, "tid": 1381170, "ts": 1514657862652.483,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862762.855, "dur": 108.338,
    "args": {
      "External id": 2488411,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862915.299, "dur": 43.137,
    "args": {
      "External id": 2488412,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657862975.236, "dur": 94.987,
    "args": {
      "External id": 2488413,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863084.909, "dur": 34.842,
    "args": {
      "External id": 2488414,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863126.524, "dur": 49.168,
    "args": {
      "External id": 2488415,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863184.538, "dur": 30.565,
    "args": {
      "External id": 2488416,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863222.898, "dur": 43.686,
    "args": {
      "External id": 2488417,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863292.939, "dur": 25.936,
    "args": {
      "External id": 2488418,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863338.739, "dur": 31.448,
    "args": {
      "External id": 2488419,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863389.627, "dur": 20.707,
    "args": {
      "External id": 2488420,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863424.284, "dur": 16.916,
    "args": {
      "External id": 2488421,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863451.192, "dur": 28.306,
    "args": {
      "External id": 2488422,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863482.585, "dur": 32.646,
    "args": {
      "External id": 2488423,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863558.560, "dur": 173.849,
    "args": {
      "External id": 2488424,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863636.109, "dur": 10.116,
    "args": {
      "External id": 2488425,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863648.098, "dur": 2.035,
    "args": {
      "External id": 2488426,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863767.400, "dur": 25.267,
    "args": {
      "External id": 2488427,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863807.672, "dur": 15.308,
    "args": {
      "External id": 2488428,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863832.453, "dur": 41.117,
    "args": {
      "External id": 2488429,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863878.533, "dur": 34.946,
    "args": {
      "External id": 2488430,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863919.589, "dur": 21.696,
    "args": {
      "External id": 2488431,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657863945.394, "dur": 31.023,
    "args": {
      "External id": 2488432,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864018.506, "dur": 26.539,
    "args": {
      "External id": 2488433,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864054.845, "dur": 30.820,
    "args": {
      "External id": 2488434,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864117.339, "dur": 28.105,
    "args": {
      "External id": 2488435,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864165.236, "dur": 24.431,
    "args": {
      "External id": 2488436,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864204.501, "dur": 19.677,
    "args": {
      "External id": 2488437,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864240.593, "dur": 14.851,
    "args": {
      "External id": 2488438,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864268.218, "dur": 20.257,
    "args": {
      "External id": 2488439,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864359.767, "dur": 16.250,
    "args": {
      "External id": 2488440,"Record function id": 0, "Ev Idx": 5239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864362.827, "dur": 12.334,
    "args": {
      "External id": 2488441,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864367.073, "dur": 7.304,
    "args": {
      "External id": 2488442,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864368.711, "dur": 5.578,
    "args": {
      "External id": 2488443,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864380.067, "dur": 5.133,
    "args": {
      "External id": 2488444,"Record function id": 0, "Ev Idx": 5243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864381.698, "dur": 3.094,
    "args": {
      "External id": 2488445,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864382.645, "dur": 1.600,
    "args": {
      "External id": 2488446,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864383.299, "dur": 0.833,
    "args": {
      "External id": 2488447,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864388.564, "dur": 5.068,
    "args": {
      "External id": 2488448,"Record function id": 0, "Ev Idx": 5247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864389.865, "dur": 3.302,
    "args": {
      "External id": 2488449,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864390.847, "dur": 1.932,
    "args": {
      "External id": 2488450,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864392.039, "dur": 0.636,
    "args": {
      "External id": 2488451,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864396.850, "dur": 4.218,
    "args": {
      "External id": 2488452,"Record function id": 0, "Ev Idx": 5251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864398.244, "dur": 2.420,
    "args": {
      "External id": 2488453,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864398.892, "dur": 1.368,
    "args": {
      "External id": 2488454,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864399.478, "dur": 0.697,
    "args": {
      "External id": 2488455,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864404.205, "dur": 3.731,
    "args": {
      "External id": 2488456,"Record function id": 0, "Ev Idx": 5255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864405.320, "dur": 2.207,
    "args": {
      "External id": 2488457,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864406.077, "dur": 1.065,
    "args": {
      "External id": 2488458,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864406.541, "dur": 0.525,
    "args": {
      "External id": 2488459,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864411.014, "dur": 3.835,
    "args": {
      "External id": 2488460,"Record function id": 0, "Ev Idx": 5259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864412.218, "dur": 2.222,
    "args": {
      "External id": 2488461,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864412.737, "dur": 1.311,
    "args": {
      "External id": 2488462,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864413.431, "dur": 0.548,
    "args": {
      "External id": 2488463,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864418.057, "dur": 5.084,
    "args": {
      "External id": 2488464,"Record function id": 0, "Ev Idx": 5263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864419.348, "dur": 3.360,
    "args": {
      "External id": 2488465,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864420.125, "dur": 2.193,
    "args": {
      "External id": 2488466,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864421.707, "dur": 0.522,
    "args": {
      "External id": 2488467,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864426.241, "dur": 9.450,
    "args": {
      "External id": 2488468,"Record function id": 0, "Ev Idx": 5267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864427.254, "dur": 7.993,
    "args": {
      "External id": 2488469,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864431.605, "dur": 3.249,
    "args": {
      "External id": 2488470,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864432.105, "dur": 2.685,
    "args": {
      "External id": 2488471,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864438.703, "dur": 4.135,
    "args": {
      "External id": 2488472,"Record function id": 0, "Ev Idx": 5271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864440.066, "dur": 2.338,
    "args": {
      "External id": 2488473,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864440.607, "dur": 1.411,
    "args": {
      "External id": 2488474,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864441.044, "dur": 0.886,
    "args": {
      "External id": 2488475,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864446.952, "dur": 37656.825,
    "args": {
      "External id": 2488476,"Record function id": 0, "Sequence number": 24740427, "Fwd thread id": 1, "Ev Idx": 5275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864448.400, "dur": 37645.877,
    "args": {
      "External id": 2488477,"Sequence number": 24740427, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5276
    }
  },
  {
    "ph": "f", "id": 221, "pid": 1336757, "tid": 1381170, "ts": 1514657864448.400,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.3)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864477.082, "dur": 37.842,
    "args": {
      "External id": 2488478,"Record function id": 0, "Ev Idx": 5277
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.3)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864523.622, "dur": 80.180,
    "args": {
      "External id": 2488479,"Record function id": 0, "Ev Idx": 5278
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.3)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864611.715, "dur": 37474.314,
    "args": {
      "External id": 2488480,"Record function id": 0, "Ev Idx": 5279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864698.872, "dur": 7.191,
    "args": {
      "External id": 2488481,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864715.815, "dur": 4.994,
    "args": {
      "External id": 2488482,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864734.241, "dur": 36497.527,
    "args": {
      "External id": 2488483,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864747.461, "dur": 36472.403,
    "args": {
      "External id": 2488484,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864791.850, "dur": 13.402,
    "args": {
      "External id": 2488485,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864811.267, "dur": 36367.900,
    "args": {
      "External id": 2488486,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864813.784, "dur": 36364.606,
    "args": {
      "External id": 2488487,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864817.324, "dur": 5.661,
    "args": {
      "External id": 2488488,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657864826.124, "dur": 36348.696,
    "args": {
      "External id": 2488489,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657901321.290, "dur": 8.956,
    "args": {
      "External id": 2488490,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657901323.931, "dur": 5.949,
    "args": {
      "External id": 2488491,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657901360.275, "dur": 386.187,
    "args": {
      "External id": 2488492,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657901388.384, "dur": 352.944,
    "args": {
      "External id": 2488493,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5292, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657901402.212, "dur": 333.097,
    "args": {
      "External id": 2488494,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657901768.204, "dur": 2.222,
    "args": {
      "External id": 2488495,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5294, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657901828.959, "dur": 6.442,
    "args": {
      "External id": 2488496,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657901881.447, "dur": 1.417,
    "args": {
      "External id": 2488497,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657901897.538, "dur": 3.249,
    "args": {
      "External id": 2488498,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657901912.972, "dur": 0.823,
    "args": {
      "External id": 2488499,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657901925.271, "dur": 0.807,
    "args": {
      "External id": 2488500,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657901938.380, "dur": 1.052,
    "args": {
      "External id": 2488501,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657901951.653, "dur": 2.863,
    "args": {
      "External id": 2488502,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657901968.400, "dur": 1.095,
    "args": {
      "External id": 2488503,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657901979.035, "dur": 0.780,
    "args": {
      "External id": 2488504,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902119.306, "dur": 2754.376,
    "args": {
      "External id": 2488505,"Record function id": 0, "Ev Idx": 5304
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.2)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902139.307, "dur": 1058.348,
    "args": {
      "External id": 2488506,"Record function id": 0, "Ev Idx": 5305
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.2)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902152.938, "dur": 318.673,
    "args": {
      "External id": 2488507,"Record function id": 0, "Ev Idx": 5306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902239.425, "dur": 4.903,
    "args": {
      "External id": 2488508,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902247.371, "dur": 1.312,
    "args": {
      "External id": 2488509,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902252.306, "dur": 1.184,
    "args": {
      "External id": 2488510,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902255.307, "dur": 2.936,
    "args": {
      "External id": 2488511,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902259.609, "dur": 1.176,
    "args": {
      "External id": 2488512,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902262.054, "dur": 1.020,
    "args": {
      "External id": 2488513,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902269.074, "dur": 0.844,
    "args": {
      "External id": 2488514,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902271.367, "dur": 1.156,
    "args": {
      "External id": 2488515,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902273.849, "dur": 1.101,
    "args": {
      "External id": 2488516,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902276.372, "dur": 1.180,
    "args": {
      "External id": 2488517,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902297.038, "dur": 147.442,
    "args": {
      "External id": 2488518,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902312.026, "dur": 128.459,
    "args": {
      "External id": 2488519,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902330.644, "dur": 12.228,
    "args": {
      "External id": 2488520,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902345.830, "dur": 66.043,
    "args": {
      "External id": 2488521,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902348.496, "dur": 63.058,
    "args": {
      "External id": 2488522,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902352.291, "dur": 8.461,
    "args": {
      "External id": 2488523,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902362.260, "dur": 48.730,
    "args": {
      "External id": 2488524,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5323
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902591.598, "dur": 598.137,
    "args": {
      "External id": 2488525,"Record function id": 0, "Ev Idx": 5324
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.1)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902607.994, "dur": 565.923,
    "args": {
      "External id": 2488526,"Record function id": 0, "Ev Idx": 5325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902673.751, "dur": 5.445,
    "args": {
      "External id": 2488527,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902701.482, "dur": 25.972,
    "args": {
      "External id": 2488528,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902707.572, "dur": 2.583,
    "args": {
      "External id": 2488529,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902711.527, "dur": 0.429,
    "args": {
      "External id": 2488530,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902713.152, "dur": 0.436,
    "args": {
      "External id": 2488531,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902715.231, "dur": 0.365,
    "args": {
      "External id": 2488532,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902716.463, "dur": 0.305,
    "args": {
      "External id": 2488533,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902717.683, "dur": 0.291,
    "args": {
      "External id": 2488534,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902719.731, "dur": 1.891,
    "args": {
      "External id": 2488535,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902722.614, "dur": 0.239,
    "args": {
      "External id": 2488536,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902723.732, "dur": 0.646,
    "args": {
      "External id": 2488537,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902737.911, "dur": 32.051,
    "args": {
      "External id": 2488538,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902798.609, "dur": 91.727,
    "args": {
      "External id": 2488539,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902808.290, "dur": 3.111,
    "args": {
      "External id": 2488540,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902816.044, "dur": 9.586,
    "args": {
      "External id": 2488541,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902819.820, "dur": 5.415,
    "args": {
      "External id": 2488542,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902823.420, "dur": 0.653,
    "args": {
      "External id": 2488543,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902831.910, "dur": 24.983,
    "args": {
      "External id": 2488544,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902833.476, "dur": 0.681,
    "args": {
      "External id": 2488545,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902835.220, "dur": 0.483,
    "args": {
      "External id": 2488546,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902837.073, "dur": 0.594,
    "args": {
      "External id": 2488547,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902839.005, "dur": 2.295,
    "args": {
      "External id": 2488548,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902842.264, "dur": 0.558,
    "args": {
      "External id": 2488549,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902843.852, "dur": 0.400,
    "args": {
      "External id": 2488550,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902847.590, "dur": 0.555,
    "args": {
      "External id": 2488551,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902849.335, "dur": 0.277,
    "args": {
      "External id": 2488552,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902850.739, "dur": 0.580,
    "args": {
      "External id": 2488553,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902866.034, "dur": 17.336,
    "args": {
      "External id": 2488554,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902931.787, "dur": 165.723,
    "args": {
      "External id": 2488555,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902962.505, "dur": 131.351,
    "args": {
      "External id": 2488556,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5355, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657902971.575, "dur": 118.100,
    "args": {
      "External id": 2488557,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657903115.346, "dur": 1.981,
    "args": {
      "External id": 2488558,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5357, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657903205.381, "dur": 1647.816,
    "args": {
      "External id": 2488559,"Sequence number": 24740426, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5358
    }
  },
  {
    "ph": "f", "id": 222, "pid": 1336757, "tid": 1381170, "ts": 1514657903205.381,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657903324.428, "dur": 104.772,
    "args": {
      "External id": 2488560,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657903469.824, "dur": 41.267,
    "args": {
      "External id": 2488561,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657903526.198, "dur": 76.497,
    "args": {
      "External id": 2488562,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657903615.246, "dur": 36.050,
    "args": {
      "External id": 2488563,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657903660.407, "dur": 45.629,
    "args": {
      "External id": 2488564,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657903713.033, "dur": 27.116,
    "args": {
      "External id": 2488565,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657903747.933, "dur": 43.261,
    "args": {
      "External id": 2488566,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657903816.198, "dur": 24.732,
    "args": {
      "External id": 2488567,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657903859.303, "dur": 32.043,
    "args": {
      "External id": 2488568,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657903913.059, "dur": 20.227,
    "args": {
      "External id": 2488569,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657903947.045, "dur": 17.338,
    "args": {
      "External id": 2488570,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657903975.199, "dur": 70.048,
    "args": {
      "External id": 2488571,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904051.236, "dur": 37.267,
    "args": {
      "External id": 2488572,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904118.212, "dur": 173.276,
    "args": {
      "External id": 2488573,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904195.217, "dur": 6.314,
    "args": {
      "External id": 2488574,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904203.521, "dur": 3.651,
    "args": {
      "External id": 2488575,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904324.455, "dur": 26.237,
    "args": {
      "External id": 2488576,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904361.301, "dur": 16.196,
    "args": {
      "External id": 2488577,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904386.113, "dur": 37.263,
    "args": {
      "External id": 2488578,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904430.146, "dur": 37.048,
    "args": {
      "External id": 2488579,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904473.849, "dur": 21.844,
    "args": {
      "External id": 2488580,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904502.965, "dur": 48.263,
    "args": {
      "External id": 2488581,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904561.064, "dur": 24.693,
    "args": {
      "External id": 2488582,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904593.137, "dur": 31.798,
    "args": {
      "External id": 2488583,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904654.865, "dur": 30.512,
    "args": {
      "External id": 2488584,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904705.193, "dur": 25.517,
    "args": {
      "External id": 2488585,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904744.683, "dur": 17.370,
    "args": {
      "External id": 2488586,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904776.790, "dur": 15.224,
    "args": {
      "External id": 2488587,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904803.985, "dur": 17.339,
    "args": {
      "External id": 2488588,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904895.531, "dur": 14.888,
    "args": {
      "External id": 2488589,"Record function id": 0, "Ev Idx": 5388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904898.900, "dur": 10.631,
    "args": {
      "External id": 2488590,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904902.798, "dur": 5.807,
    "args": {
      "External id": 2488591,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904904.679, "dur": 3.775,
    "args": {
      "External id": 2488592,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904914.335, "dur": 4.882,
    "args": {
      "External id": 2488593,"Record function id": 0, "Ev Idx": 5392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904915.626, "dur": 3.154,
    "args": {
      "External id": 2488594,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904916.446, "dur": 1.752,
    "args": {
      "External id": 2488595,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904917.297, "dur": 0.777,
    "args": {
      "External id": 2488596,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904922.362, "dur": 5.485,
    "args": {
      "External id": 2488597,"Record function id": 0, "Ev Idx": 5396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904924.001, "dur": 3.423,
    "args": {
      "External id": 2488598,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904925.003, "dur": 2.000,
    "args": {
      "External id": 2488599,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904925.876, "dur": 1.062,
    "args": {
      "External id": 2488600,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904931.011, "dur": 4.432,
    "args": {
      "External id": 2488601,"Record function id": 0, "Ev Idx": 5400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904932.283, "dur": 2.750,
    "args": {
      "External id": 2488602,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904933.373, "dur": 1.242,
    "args": {
      "External id": 2488603,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904933.843, "dur": 0.709,
    "args": {
      "External id": 2488604,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904938.656, "dur": 4.986,
    "args": {
      "External id": 2488605,"Record function id": 0, "Ev Idx": 5404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904940.071, "dur": 3.164,
    "args": {
      "External id": 2488606,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904940.975, "dur": 1.835,
    "args": {
      "External id": 2488607,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904942.126, "dur": 0.608,
    "args": {
      "External id": 2488608,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904946.698, "dur": 3.531,
    "args": {
      "External id": 2488609,"Record function id": 0, "Ev Idx": 5408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904947.756, "dur": 2.035,
    "args": {
      "External id": 2488610,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904948.235, "dur": 1.126,
    "args": {
      "External id": 2488611,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904948.675, "dur": 0.577,
    "args": {
      "External id": 2488612,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904953.483, "dur": 4.818,
    "args": {
      "External id": 2488613,"Record function id": 0, "Ev Idx": 5412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904954.432, "dur": 3.450,
    "args": {
      "External id": 2488614,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904955.068, "dur": 2.340,
    "args": {
      "External id": 2488615,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904955.330, "dur": 1.997,
    "args": {
      "External id": 2488616,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904961.444, "dur": 3.969,
    "args": {
      "External id": 2488617,"Record function id": 0, "Ev Idx": 5416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904962.575, "dur": 2.403,
    "args": {
      "External id": 2488618,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904963.283, "dur": 1.254,
    "args": {
      "External id": 2488619,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904963.530, "dur": 0.905,
    "args": {
      "External id": 2488620,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904968.461, "dur": 3.840,
    "args": {
      "External id": 2488621,"Record function id": 0, "Ev Idx": 5420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904969.479, "dur": 2.375,
    "args": {
      "External id": 2488622,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904969.959, "dur": 1.495,
    "args": {
      "External id": 2488623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904970.394, "dur": 0.946,
    "args": {
      "External id": 2488624,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904976.047, "dur": 37166.963,
    "args": {
      "External id": 2488625,"Record function id": 0, "Sequence number": 24740425, "Fwd thread id": 1, "Ev Idx": 5424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657904977.612, "dur": 37156.074,
    "args": {
      "External id": 2488626,"Sequence number": 24740425, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5425
    }
  },
  {
    "ph": "f", "id": 223, "pid": 1336757, "tid": 1381170, "ts": 1514657904977.612,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.2)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657905040.659, "dur": 39.750,
    "args": {
      "External id": 2488627,"Record function id": 0, "Ev Idx": 5426
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.2)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657905089.009, "dur": 70.088,
    "args": {
      "External id": 2488628,"Record function id": 0, "Ev Idx": 5427
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.2)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657905165.300, "dur": 36960.279,
    "args": {
      "External id": 2488629,"Record function id": 0, "Ev Idx": 5428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657905253.022, "dur": 7.062,
    "args": {
      "External id": 2488630,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657905270.105, "dur": 4.969,
    "args": {
      "External id": 2488631,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657905289.720, "dur": 35970.573,
    "args": {
      "External id": 2488632,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657905302.371, "dur": 35949.493,
    "args": {
      "External id": 2488633,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657905354.273, "dur": 13.598,
    "args": {
      "External id": 2488634,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657905373.817, "dur": 35841.489,
    "args": {
      "External id": 2488635,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657905376.161, "dur": 35838.423,
    "args": {
      "External id": 2488636,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657905379.707, "dur": 5.203,
    "args": {
      "External id": 2488637,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657905386.667, "dur": 35824.310,
    "args": {
      "External id": 2488638,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657941350.470, "dur": 10.372,
    "args": {
      "External id": 2488639,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657941353.331, "dur": 7.117,
    "args": {
      "External id": 2488640,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657941387.599, "dur": 401.135,
    "args": {
      "External id": 2488641,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657941416.081, "dur": 367.791,
    "args": {
      "External id": 2488642,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5441, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657941426.886, "dur": 351.077,
    "args": {
      "External id": 2488643,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657941809.737, "dur": 2.646,
    "args": {
      "External id": 2488644,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5443, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657941871.102, "dur": 6.434,
    "args": {
      "External id": 2488645,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657941920.451, "dur": 3.263,
    "args": {
      "External id": 2488646,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657941943.443, "dur": 1.315,
    "args": {
      "External id": 2488647,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657941956.756, "dur": 1.081,
    "args": {
      "External id": 2488648,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657941969.252, "dur": 1.428,
    "args": {
      "External id": 2488649,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657941980.984, "dur": 36.946,
    "args": {
      "External id": 2488650,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942035.217, "dur": 1.709,
    "args": {
      "External id": 2488651,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942047.767, "dur": 0.916,
    "args": {
      "External id": 2488652,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942061.986, "dur": 0.818,
    "args": {
      "External id": 2488653,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942158.143, "dur": 2749.846,
    "args": {
      "External id": 2488654,"Record function id": 0, "Ev Idx": 5453
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.1)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942181.598, "dur": 1056.799,
    "args": {
      "External id": 2488655,"Record function id": 0, "Ev Idx": 5454
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.1)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942194.586, "dur": 324.871,
    "args": {
      "External id": 2488656,"Record function id": 0, "Ev Idx": 5455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942286.000, "dur": 4.510,
    "args": {
      "External id": 2488657,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942296.705, "dur": 3.126,
    "args": {
      "External id": 2488658,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942302.040, "dur": 0.757,
    "args": {
      "External id": 2488659,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942304.499, "dur": 0.963,
    "args": {
      "External id": 2488660,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942306.998, "dur": 0.889,
    "args": {
      "External id": 2488661,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942309.125, "dur": 1.036,
    "args": {
      "External id": 2488662,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942311.846, "dur": 1.136,
    "args": {
      "External id": 2488663,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942314.924, "dur": 0.951,
    "args": {
      "External id": 2488664,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942317.194, "dur": 0.881,
    "args": {
      "External id": 2488665,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942319.674, "dur": 2.597,
    "args": {
      "External id": 2488666,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942339.865, "dur": 153.068,
    "args": {
      "External id": 2488667,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942358.734, "dur": 130.303,
    "args": {
      "External id": 2488668,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942373.905, "dur": 11.561,
    "args": {
      "External id": 2488669,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942388.692, "dur": 70.303,
    "args": {
      "External id": 2488670,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942392.458, "dur": 66.134,
    "args": {
      "External id": 2488671,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942396.285, "dur": 5.554,
    "args": {
      "External id": 2488672,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942403.707, "dur": 54.356,
    "args": {
      "External id": 2488673,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5472
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::backward_prefetch for model.layers.0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942652.229, "dur": 578.461,
    "args": {
      "External id": 2488674,"Record function id": 0, "Ev Idx": 5473
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.0)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942671.081, "dur": 547.568,
    "args": {
      "External id": 2488675,"Record function id": 0, "Ev Idx": 5474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942742.044, "dur": 5.644,
    "args": {
      "External id": 2488676,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942762.857, "dur": 27.949,
    "args": {
      "External id": 2488677,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942767.395, "dur": 1.445,
    "args": {
      "External id": 2488678,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942770.919, "dur": 0.636,
    "args": {
      "External id": 2488679,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942773.420, "dur": 0.306,
    "args": {
      "External id": 2488680,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942774.820, "dur": 0.606,
    "args": {
      "External id": 2488681,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942777.537, "dur": 0.775,
    "args": {
      "External id": 2488682,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942779.677, "dur": 2.815,
    "args": {
      "External id": 2488683,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942783.607, "dur": 0.561,
    "args": {
      "External id": 2488684,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942785.861, "dur": 0.347,
    "args": {
      "External id": 2488685,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942787.416, "dur": 0.505,
    "args": {
      "External id": 2488686,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942801.071, "dur": 31.992,
    "args": {
      "External id": 2488687,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942862.595, "dur": 95.110,
    "args": {
      "External id": 2488688,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 5487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942872.434, "dur": 3.180,
    "args": {
      "External id": 2488689,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942880.205, "dur": 9.727,
    "args": {
      "External id": 2488690,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942884.207, "dur": 5.198,
    "args": {
      "External id": 2488691,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 5490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942887.694, "dur": 0.436,
    "args": {
      "External id": 2488692,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 5491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942896.747, "dur": 26.878,
    "args": {
      "External id": 2488693,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942898.894, "dur": 0.417,
    "args": {
      "External id": 2488694,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942901.163, "dur": 0.428,
    "args": {
      "External id": 2488695,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942903.062, "dur": 2.206,
    "args": {
      "External id": 2488696,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942906.839, "dur": 0.436,
    "args": {
      "External id": 2488697,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942908.952, "dur": 0.332,
    "args": {
      "External id": 2488698,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942910.239, "dur": 0.446,
    "args": {
      "External id": 2488699,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942912.119, "dur": 0.440,
    "args": {
      "External id": 2488700,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942914.423, "dur": 0.429,
    "args": {
      "External id": 2488701,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942916.258, "dur": 0.565,
    "args": {
      "External id": 2488702,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657942932.872, "dur": 17.412,
    "args": {
      "External id": 2488703,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943037.788, "dur": 112.093,
    "args": {
      "External id": 2488704,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 5503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943060.868, "dur": 85.993,
    "args": {
      "External id": 2488705,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5504, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943070.490, "dur": 72.231,
    "args": {
      "External id": 2488706,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 5505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943164.867, "dur": 1.841,
    "args": {
      "External id": 2488707,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5506, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943245.544, "dur": 1642.724,
    "args": {
      "External id": 2488708,"Sequence number": 24740424, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5507
    }
  },
  {
    "ph": "f", "id": 224, "pid": 1336757, "tid": 1381170, "ts": 1514657943245.544,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943360.708, "dur": 101.598,
    "args": {
      "External id": 2488709,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943501.730, "dur": 59.592,
    "args": {
      "External id": 2488710,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943579.448, "dur": 52.783,
    "args": {
      "External id": 2488711,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943641.576, "dur": 33.002,
    "args": {
      "External id": 2488712,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943681.267, "dur": 45.495,
    "args": {
      "External id": 2488713,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943735.601, "dur": 27.943,
    "args": {
      "External id": 2488714,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943770.737, "dur": 41.731,
    "args": {
      "External id": 2488715,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943837.600, "dur": 29.162,
    "args": {
      "External id": 2488716,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943884.325, "dur": 30.425,
    "args": {
      "External id": 2488717,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943936.101, "dur": 19.206,
    "args": {
      "External id": 2488718,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657943968.936, "dur": 52.936,
    "args": {
      "External id": 2488719,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944036.160, "dur": 36.091,
    "args": {
      "External id": 2488720,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944075.960, "dur": 35.461,
    "args": {
      "External id": 2488721,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944140.645, "dur": 173.039,
    "args": {
      "External id": 2488722,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944219.201, "dur": 6.478,
    "args": {
      "External id": 2488723,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944228.285, "dur": 2.564,
    "args": {
      "External id": 2488724,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944347.984, "dur": 25.620,
    "args": {
      "External id": 2488725,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944384.810, "dur": 16.988,
    "args": {
      "External id": 2488726,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944411.468, "dur": 34.127,
    "args": {
      "External id": 2488727,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944452.169, "dur": 35.986,
    "args": {
      "External id": 2488728,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944495.062, "dur": 22.338,
    "args": {
      "External id": 2488729,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944522.972, "dur": 50.289,
    "args": {
      "External id": 2488730,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944585.008, "dur": 23.715,
    "args": {
      "External id": 2488731,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944617.148, "dur": 46.551,
    "args": {
      "External id": 2488732,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944688.133, "dur": 24.760,
    "args": {
      "External id": 2488733,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944736.698, "dur": 25.815,
    "args": {
      "External id": 2488734,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944781.620, "dur": 17.682,
    "args": {
      "External id": 2488735,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944815.025, "dur": 15.826,
    "args": {
      "External id": 2488736,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944843.815, "dur": 17.977,
    "args": {
      "External id": 2488737,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944929.954, "dur": 15.096,
    "args": {
      "External id": 2488738,"Record function id": 0, "Ev Idx": 5537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944933.462, "dur": 10.771,
    "args": {
      "External id": 2488739,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944937.528, "dur": 5.700,
    "args": {
      "External id": 2488740,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944939.027, "dur": 4.087,
    "args": {
      "External id": 2488741,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944948.909, "dur": 5.552,
    "args": {
      "External id": 2488742,"Record function id": 0, "Ev Idx": 5541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944950.483, "dur": 3.532,
    "args": {
      "External id": 2488743,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944951.452, "dur": 1.964,
    "args": {
      "External id": 2488744,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944952.265, "dur": 1.056,
    "args": {
      "External id": 2488745,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944957.912, "dur": 4.591,
    "args": {
      "External id": 2488746,"Record function id": 0, "Ev Idx": 5545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944959.599, "dur": 2.493,
    "args": {
      "External id": 2488747,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944960.265, "dur": 1.388,
    "args": {
      "External id": 2488748,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944960.727, "dur": 0.856,
    "args": {
      "External id": 2488749,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944965.615, "dur": 4.014,
    "args": {
      "External id": 2488750,"Record function id": 0, "Ev Idx": 5549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944966.750, "dur": 2.427,
    "args": {
      "External id": 2488751,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944967.441, "dur": 1.306,
    "args": {
      "External id": 2488752,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944967.741, "dur": 0.942,
    "args": {
      "External id": 2488753,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944975.832, "dur": 39.866,
    "args": {
      "External id": 2488754,"Record function id": 0, "Ev Idx": 5553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944977.197, "dur": 37.338,
    "args": {
      "External id": 2488755,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944977.878, "dur": 35.447,
    "args": {
      "External id": 2488756,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657944978.203, "dur": 34.530,
    "args": {
      "External id": 2488757,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945021.649, "dur": 6.100,
    "args": {
      "External id": 2488758,"Record function id": 0, "Ev Idx": 5557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945023.347, "dur": 3.909,
    "args": {
      "External id": 2488759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945024.642, "dur": 2.159,
    "args": {
      "External id": 2488760,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945025.403, "dur": 1.288,
    "args": {
      "External id": 2488761,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945031.143, "dur": 4.229,
    "args": {
      "External id": 2488762,"Record function id": 0, "Ev Idx": 5561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945032.513, "dur": 2.424,
    "args": {
      "External id": 2488763,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945033.311, "dur": 1.214,
    "args": {
      "External id": 2488764,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945033.682, "dur": 0.754,
    "args": {
      "External id": 2488765,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945038.443, "dur": 4.748,
    "args": {
      "External id": 2488766,"Record function id": 0, "Ev Idx": 5565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945039.947, "dur": 2.801,
    "args": {
      "External id": 2488767,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945040.743, "dur": 1.557,
    "args": {
      "External id": 2488768,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945041.262, "dur": 0.942,
    "args": {
      "External id": 2488769,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945046.371, "dur": 4.270,
    "args": {
      "External id": 2488770,"Record function id": 0, "Ev Idx": 5569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945047.749, "dur": 2.475,
    "args": {
      "External id": 2488771,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945048.382, "dur": 1.305,
    "args": {
      "External id": 2488772,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945048.920, "dur": 0.644,
    "args": {
      "External id": 2488773,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945054.540, "dur": 36125.458,
    "args": {
      "External id": 2488774,"Record function id": 0, "Sequence number": 24740423, "Fwd thread id": 1, "Ev Idx": 5573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945056.131, "dur": 36115.971,
    "args": {
      "External id": 2488775,"Sequence number": 24740423, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5574
    }
  },
  {
    "ph": "f", "id": 225, "pid": 1336757, "tid": 1381170, "ts": 1514657945056.131,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.1)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945083.612, "dur": 36.617,
    "args": {
      "External id": 2488776,"Record function id": 0, "Ev Idx": 5575
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.1)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945127.872, "dur": 64.641,
    "args": {
      "External id": 2488777,"Record function id": 0, "Ev Idx": 5576
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.1)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945198.003, "dur": 35966.933,
    "args": {
      "External id": 2488778,"Record function id": 0, "Ev Idx": 5577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945282.570, "dur": 6.486,
    "args": {
      "External id": 2488779,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945298.320, "dur": 4.970,
    "args": {
      "External id": 2488780,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945317.042, "dur": 35032.483,
    "args": {
      "External id": 2488781,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945329.701, "dur": 35007.795,
    "args": {
      "External id": 2488782,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945374.586, "dur": 14.332,
    "args": {
      "External id": 2488783,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945395.078, "dur": 34904.200,
    "args": {
      "External id": 2488784,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945397.887, "dur": 34900.588,
    "args": {
      "External id": 2488785,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945401.898, "dur": 5.917,
    "args": {
      "External id": 2488786,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657945409.430, "dur": 34885.377,
    "args": {
      "External id": 2488787,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657980437.282, "dur": 10.029,
    "args": {
      "External id": 2488788,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657980441.242, "dur": 5.728,
    "args": {
      "External id": 2488789,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657980476.019, "dur": 364.450,
    "args": {
      "External id": 2488790,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657980502.315, "dur": 333.194,
    "args": {
      "External id": 2488791,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5590, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514657980513.474, "dur": 315.917,
    "args": {
      "External id": 2488792,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514657980861.332, "dur": 2.237,
    "args": {
      "External id": 2488793,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5592, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657980921.893, "dur": 8.947,
    "args": {
      "External id": 2488794,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657980973.212, "dur": 1.373,
    "args": {
      "External id": 2488795,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981025.434, "dur": 1.924,
    "args": {
      "External id": 2488796,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981040.900, "dur": 1.039,
    "args": {
      "External id": 2488797,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981054.046, "dur": 2.737,
    "args": {
      "External id": 2488798,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981066.549, "dur": 1.278,
    "args": {
      "External id": 2488799,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981079.245, "dur": 1.194,
    "args": {
      "External id": 2488800,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981091.360, "dur": 1.276,
    "args": {
      "External id": 2488801,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981102.490, "dur": 3.035,
    "args": {
      "External id": 2488802,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981194.860, "dur": 2218.408,
    "args": {
      "External id": 2488803,"Record function id": 0, "Ev Idx": 5602
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_backward (model.layers.0)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981213.517, "dur": 493.013,
    "args": {
      "External id": 2488804,"Record function id": 0, "Ev Idx": 5603
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.0)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981226.155, "dur": 353.622,
    "args": {
      "External id": 2488805,"Record function id": 0, "Ev Idx": 5604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981326.782, "dur": 4.137,
    "args": {
      "External id": 2488806,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981334.204, "dur": 1.355,
    "args": {
      "External id": 2488807,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981337.280, "dur": 1.104,
    "args": {
      "External id": 2488808,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981340.026, "dur": 1.099,
    "args": {
      "External id": 2488809,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981342.760, "dur": 0.934,
    "args": {
      "External id": 2488810,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981346.433, "dur": 1.177,
    "args": {
      "External id": 2488811,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981349.509, "dur": 0.926,
    "args": {
      "External id": 2488812,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981351.939, "dur": 3.002,
    "args": {
      "External id": 2488813,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981356.644, "dur": 1.122,
    "args": {
      "External id": 2488814,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981360.218, "dur": 0.966,
    "args": {
      "External id": 2488815,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981378.517, "dur": 170.966,
    "args": {
      "External id": 2488816,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981396.904, "dur": 147.369,
    "args": {
      "External id": 2488817,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 5616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981411.342, "dur": 12.262,
    "args": {
      "External id": 2488818,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981427.003, "dur": 74.108,
    "args": {
      "External id": 2488819,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 5618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981430.056, "dur": 70.545,
    "args": {
      "External id": 2488820,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 5619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981438.169, "dur": 6.074,
    "args": {
      "External id": 2488821,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981445.995, "dur": 53.950,
    "args": {
      "External id": 2488822,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 5621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981714.509, "dur": 1673.837,
    "args": {
      "External id": 2488823,"Sequence number": 24740422, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5622
    }
  },
  {
    "ph": "f", "id": 226, "pid": 1336757, "tid": 1381170, "ts": 1514657981714.509,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981826.645, "dur": 108.281,
    "args": {
      "External id": 2488824,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [5632, 1], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 5623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_add_mul_rsub_silu_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657981974.034, "dur": 83.663,
    "args": {
      "External id": 2488825,"kernel_hash": "cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qt/cqtv52ovuip3ol5r6wg244twcee7ut4ombdtlv3jt7eknkh3yxxu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [65536, 5632], [16, 4096, 5632], [16, 4096, 5632], [16, 4096, 5632], []], "Ev Idx": 5624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::bmm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982079.411, "dur": 57.830,
    "args": {
      "External id": 2488826,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[0, 1, 2048], [0, 5632, 1], [11534336, 5632, 1]], "Input Dims": [[1, 2048, 65536], [1, 65536, 5632], [1, 2048, 5632]], "Ev Idx": 5625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982147.247, "dur": 34.102,
    "args": {
      "External id": 2488827,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982188.883, "dur": 47.419,
    "args": {
      "External id": 2488828,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982246.916, "dur": 29.219,
    "args": {
      "External id": 2488829,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 5632], [2048, 1], [2048, 1]], "Input Dims": [[5632, 65536], [65536, 2048], [5632, 2048]], "Ev Idx": 5628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982284.182, "dur": 44.114,
    "args": {
      "External id": 2488830,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 5629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982352.780, "dur": 27.532,
    "args": {
      "External id": 2488831,"kernel_hash": "ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/t6/ct6pdrwin6gpbbhb3ubmfazw3qma3gzsewwbv4culi3mwmzponsp.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 5630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982397.777, "dur": 31.399,
    "args": {
      "External id": 2488832,"kernel_hash": "cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "True", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/z6/cz6ucetbngq4jgbopxnyfp6gcid5s5ztsv5ncyzjsjt7tfq7ipcf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982467.312, "dur": 21.693,
    "args": {
      "External id": 2488833,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982502.497, "dur": 16.165,
    "args": {
      "External id": 2488834,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982543.698, "dur": 37.548,
    "args": {
      "External id": 2488835,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982585.596, "dur": 35.655,
    "args": {
      "External id": 2488836,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_backward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982650.720, "dur": 170.568,
    "args": {
      "External id": 2488837,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar", "long int"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [131072, 4096, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [1]], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 32, 4096], [16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [2]], "Ev Idx": 5636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982728.445, "dur": 6.205,
    "args": {
      "External id": 2488838,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982736.354, "dur": 2.398,
    "args": {
      "External id": 2488839,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982855.333, "dur": 26.755,
    "args": {
      "External id": 2488840,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982893.851, "dur": 16.652,
    "args": {
      "External id": 2488841,"kernel_hash": "cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "True"], "kernel_file": "/tmp/torchinductor_cvm/p6/cp6j6nhpwjd6mqsefl44gnnjkpkrjt2lnxslguyz45haxtbxbto3.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982919.112, "dur": 34.080,
    "args": {
      "External id": 2488842,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657982961.601, "dur": 73.323,
    "args": {
      "External id": 2488843,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983045.707, "dur": 25.942,
    "args": {
      "External id": 2488844,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983077.580, "dur": 32.628,
    "args": {
      "External id": 2488845,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983116.570, "dur": 20.968,
    "args": {
      "External id": 2488846,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 2048], [2048, 1], [2048, 1]], "Input Dims": [[2048, 65536], [65536, 2048], [2048, 2048]], "Ev Idx": 5645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983148.387, "dur": 30.706,
    "args": {
      "External id": 2488847,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 5646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_4", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983197.028, "dur": 25.326,
    "args": {
      "External id": 2488848,"kernel_hash": "cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qs/cqsmoyietljwwem74pqj7rw77h2baol67y2jdqdgbsq7tblfkrf4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [65536, 2048], []], "Ev Idx": 5647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_bwd_kernel_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983239.382, "dur": 26.094,
    "args": {
      "External id": 2488849,"kernel_hash": "clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "65536", "2048", "1", "497", "132", "True", "2048", "False", "False", "True", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/lr/clrhf26bqgdimbaet5vq7dtthfchb2oh7voorngyplnz6tf2aame.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [2048], [65536, 2048], [65536, 2048], [132, 2048], [65536], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 5648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_red_fused_sum_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983280.149, "dur": 19.459,
    "args": {
      "External id": 2488850,"kernel_hash": "cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl", "grid": "grid(4096,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "4096", "66"], "kernel_file": "/tmp/torchinductor_cvm/wg/cwgmcdkxcf3qaknrmuortliovuosbdico7imkwflk5thajnwoiwl.py", "kernel_backend": "triton", "Input type": ["float", "float", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [4096, 1, 2048], [], []], "Input Dims": [[132, 2048], [1, 2048, 2], [], []], "Ev Idx": 5649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_per_fused__to_copy_sum_3", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983312.971, "dur": 16.193,
    "args": {
      "External id": 2488851,"kernel_hash": "crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd", "grid": "grid(2048,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "2048", "2"], "kernel_file": "/tmp/torchinductor_cvm/rq/crqtfcpyfpzzdibd36dgyl26ixald2ez2onww6lldxkvjtmbpczd.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 2048], [2048, 1], [], []], "Input Dims": [[1, 2048, 2], [1, 2048], [], []], "Ev Idx": 5650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_5", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983340.973, "dur": 17.682,
    "args": {
      "External id": 2488852,"kernel_hash": "cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/em/cemjdqi5ghk3i77b2rg37v3lyvjbugp7r7bobzjsarysugcnwrjj.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], [16, 4096, 2048], []], "Ev Idx": 5651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983438.897, "dur": 15.820,
    "args": {
      "External id": 2488853,"Record function id": 0, "Ev Idx": 5652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983442.476, "dur": 11.367,
    "args": {
      "External id": 2488854,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983446.854, "dur": 5.929,
    "args": {
      "External id": 2488855,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983448.571, "dur": 4.090,
    "args": {
      "External id": 2488856,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983458.528, "dur": 5.110,
    "args": {
      "External id": 2488857,"Record function id": 0, "Ev Idx": 5656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983460.354, "dur": 2.831,
    "args": {
      "External id": 2488858,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983461.037, "dur": 1.662,
    "args": {
      "External id": 2488859,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983461.841, "dur": 0.784,
    "args": {
      "External id": 2488860,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983466.801, "dur": 4.118,
    "args": {
      "External id": 2488861,"Record function id": 0, "Ev Idx": 5660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983468.212, "dur": 2.284,
    "args": {
      "External id": 2488862,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983468.725, "dur": 1.340,
    "args": {
      "External id": 2488863,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983469.200, "dur": 0.801,
    "args": {
      "External id": 2488864,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[2048]], "Ev Idx": 5663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983474.138, "dur": 4.444,
    "args": {
      "External id": 2488865,"Record function id": 0, "Ev Idx": 5664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983475.562, "dur": 2.593,
    "args": {
      "External id": 2488866,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983476.325, "dur": 1.417,
    "args": {
      "External id": 2488867,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983476.843, "dur": 0.834,
    "args": {
      "External id": 2488868,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 5667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983481.681, "dur": 6.106,
    "args": {
      "External id": 2488869,"Record function id": 0, "Ev Idx": 5668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983483.078, "dur": 4.298,
    "args": {
      "External id": 2488870,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983483.706, "dur": 3.287,
    "args": {
      "External id": 2488871,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983484.051, "dur": 2.877,
    "args": {
      "External id": 2488872,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983490.889, "dur": 4.251,
    "args": {
      "External id": 2488873,"Record function id": 0, "Ev Idx": 5672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983492.308, "dur": 2.413,
    "args": {
      "External id": 2488874,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983492.973, "dur": 1.323,
    "args": {
      "External id": 2488875,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983493.410, "dur": 0.785,
    "args": {
      "External id": 2488876,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983498.511, "dur": 4.575,
    "args": {
      "External id": 2488877,"Record function id": 0, "Ev Idx": 5676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983500.242, "dur": 2.434,
    "args": {
      "External id": 2488878,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983500.843, "dur": 1.451,
    "args": {
      "External id": 2488879,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983501.409, "dur": 0.785,
    "args": {
      "External id": 2488880,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983506.156, "dur": 4.623,
    "args": {
      "External id": 2488881,"Record function id": 0, "Ev Idx": 5680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983508.018, "dur": 2.354,
    "args": {
      "External id": 2488882,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983508.672, "dur": 1.304,
    "args": {
      "External id": 2488883,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983509.058, "dur": 0.813,
    "args": {
      "External id": 2488884,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 5683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983513.859, "dur": 4.066,
    "args": {
      "External id": 2488885,"Record function id": 0, "Ev Idx": 5684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983515.106, "dur": 2.396,
    "args": {
      "External id": 2488886,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983515.654, "dur": 1.460,
    "args": {
      "External id": 2488887,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983516.227, "dur": 0.767,
    "args": {
      "External id": 2488888,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 5687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983521.777, "dur": 36048.997,
    "args": {
      "External id": 2488889,"Record function id": 0, "Sequence number": 24740421, "Fwd thread id": 1, "Ev Idx": 5688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983522.779, "dur": 36038.571,
    "args": {
      "External id": 2488890,"Sequence number": 24740421, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5689
    }
  },
  {
    "ph": "f", "id": 227, "pid": 1336757, "tid": 1381170, "ts": 1514657983522.779,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate (model.layers.0)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983567.698, "dur": 38.958,
    "args": {
      "External id": 2488891,"Record function id": 0, "Ev Idx": 5690
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard (model.layers.0)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983614.570, "dur": 87.894,
    "args": {
      "External id": 2488892,"Record function id": 0, "Ev Idx": 5691
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce (model.layers.0)", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983711.691, "dur": 35841.411,
    "args": {
      "External id": 2488893,"Record function id": 0, "Ev Idx": 5692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983802.466, "dur": 10.483,
    "args": {
      "External id": 2488894,"Record function id": 0, "Concrete Inputs": ["[51384320]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983823.275, "dur": 5.316,
    "args": {
      "External id": 2488895,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 5694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983841.930, "dur": 34889.072,
    "args": {
      "External id": 2488896,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983854.664, "dur": 34867.901,
    "args": {
      "External id": 2488897,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[[1], [2048, 1], [2048, 1], [2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], [], [], [6423040, 1]], "Input Dims": [[[2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], [], [], [8, 6423040]], "Ev Idx": 5696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983907.815, "dur": 13.554,
    "args": {
      "External id": 2488898,"Record function id": 0, "Concrete Inputs": ["[12602]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983928.567, "dur": 34754.822,
    "args": {
      "External id": 2488899,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], [], []], "Ev Idx": 5698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983931.186, "dur": 34751.474,
    "args": {
      "External id": 2488900,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[12602], [], [], [], [], [], []], "Ev Idx": 5699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983935.827, "dur": 4.735,
    "args": {
      "External id": 2488901,"Record function id": 0, "Concrete Inputs": ["[12602]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514657983942.339, "dur": 34737.106,
    "args": {
      "External id": 2488902,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[12602], [12602], []], "Ev Idx": 5701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514658018819.888, "dur": 8.329,
    "args": {
      "External id": 2488903,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[51384320], [], [], [], [], []], "Ev Idx": 5702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514658018822.621, "dur": 5.290,
    "args": {
      "External id": 2488904,"Record function id": 0, "Concrete Inputs": ["[6423040]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514658018858.130, "dur": 388.250,
    "args": {
      "External id": 2488905,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[6423040], [51384320], [], [], [], []], "Ev Idx": 5704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514658018883.796, "dur": 358.046,
    "args": {
      "External id": 2488906,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 6423040, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[51384320], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5705, "In msg nelems": 51384320
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514658018898.084, "dur": 337.441,
    "args": {
      "External id": 2488907,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[51384320]], "Ev Idx": 5706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019269.387, "dur": 2.569,
    "args": {
      "External id": 2488908,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5707, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019334.268, "dur": 6.562,
    "args": {
      "External id": 2488909,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019383.639, "dur": 3.886,
    "args": {
      "External id": 2488910,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019403.558, "dur": 1.559,
    "args": {
      "External id": 2488911,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "524544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019417.299, "dur": 0.986,
    "args": {
      "External id": 2488912,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1048832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019430.103, "dur": 0.982,
    "args": {
      "External id": 2488913,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "1573120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019442.015, "dur": 2.742,
    "args": {
      "External id": 2488914,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019456.555, "dur": 0.948,
    "args": {
      "External id": 2488915,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "2097664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019469.258, "dur": 0.807,
    "args": {
      "External id": 2488916,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "3539456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019479.133, "dur": 0.730,
    "args": {
      "External id": 2488917,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "4981248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019587.176, "dur": 281.908,
    "args": {
      "External id": 2488918,"Record function id": 0, "Sequence number": 24740420, "Fwd thread id": 1, "Ev Idx": 5717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunctionBackward", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019590.133, "dur": 270.314,
    "args": {
      "External id": 2488919,"Sequence number": 24740420, "Fwd thread id": 1, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 5718
    }
  },
  {
    "ph": "f", "id": 228, "pid": 1336757, "tid": 1381170, "ts": 1514658019590.133,
    "cat": "fwdbwd", "name": "fwdbwd", "bp": "e"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_dense_backward_0", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019713.752, "dur": 46.693,
    "args": {
      "External id": 2488920,"kernel_hash": "ciduma7cu5eg2vb7pjzy37skdgvzi7kewhtlfbbosvucja2bojsj", "grid": "grid(65536000,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "65536000"], "kernel_file": "/tmp/torchinductor_cvm/id/ciduma7cu5eg2vb7pjzy37skdgvzi7kewhtlfbbosvucja2bojsj.py", "kernel_backend": "triton", "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 5719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_dense_backward_1", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019775.841, "dur": 28.591,
    "args": {
      "External id": 2488921,"kernel_hash": "caujkv5qkif2iwziam2o6fi3v6fyh3htmhp45aeprztex4h267rb", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/au/caujkv5qkif2iwziam2o6fi3v6fyh3htmhp45aeprztex4h267rb.py", "kernel_backend": "triton", "Input type": ["long int", "c10::BFloat16", "float", "Scalar"], "Input Strides": [[4096, 1], [8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096], [16, 4096, 2048], [32000, 2048], []], "Ev Idx": 5720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_dense_backward_2", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019822.966, "dur": 22.788,
    "args": {
      "External id": 2488922,"kernel_hash": "cfq2gzp6xuirheiddwf3ictuku4xly4uz6pgv7jhaatakl55p4ao", "grid": "grid(65536000,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "65536000"], "kernel_file": "/tmp/torchinductor_cvm/fq/cfq2gzp6xuirheiddwf3ictuku4xly4uz6pgv7jhaatakl55p4ao.py", "kernel_backend": "triton", "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 5721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019881.988, "dur": 16.322,
    "args": {
      "External id": 2488923,"Record function id": 0, "Ev Idx": 5722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "torch::autograd::AccumulateGrad", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019884.823, "dur": 12.574,
    "args": {
      "External id": 2488924,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 5723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019888.809, "dur": 7.719,
    "args": {
      "External id": 2488925,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 5724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019890.252, "dur": 6.159,
    "args": {
      "External id": 2488926,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 5725
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::root_post_backward_callback", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019920.614, "dur": 7204.086,
    "args": {
      "External id": 2488927,"Record function id": 0, "Ev Idx": 5726
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_accumulate", "pid": 1336757, "tid": 1381170,
    "ts": 1514658019938.214, "dur": 90.281,
    "args": {
      "External id": 2488928,"Record function id": 0, "Ev Idx": 5727
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reshard", "pid": 1336757, "tid": 1381170,
    "ts": 1514658020036.866, "dur": 328.878,
    "args": {
      "External id": 2488929,"Record function id": 0, "Ev Idx": 5728
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_backward_reduce", "pid": 1336757, "tid": 1381170,
    "ts": 1514658020372.331, "dur": 6436.902,
    "args": {
      "External id": 2488930,"Record function id": 0, "Ev Idx": 5729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514658020497.230, "dur": 7.592,
    "args": {
      "External id": 2488931,"Record function id": 0, "Concrete Inputs": ["[336611328]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1381170,
    "ts": 1514658020515.436, "dur": 7.826,
    "args": {
      "External id": 2488932,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[336611328], []], "Ev Idx": 5731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514658020558.243, "dur": 4903.159,
    "args": {
      "External id": 2488933,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[], [], [], [42076416, 1]], "Input Dims": [[], [], [], [8, 42076416]], "Ev Idx": 5732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_chunk_cat", "pid": 1336757, "tid": 1381170,
    "ts": 1514658020579.837, "dur": 4868.475,
    "args": {
      "External id": 2488934,"Record function id": 0, "Concrete Inputs": ["", "0", "8", ""], "Input type": ["TensorList", "Scalar", "Scalar", "float"], "Input Strides": [[], [], [], [42076416, 1]], "Input Dims": [[], [], [], [8, 42076416]], "Ev Idx": 5733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514658020922.202, "dur": 20.935,
    "args": {
      "External id": 2488935,"Record function id": 0, "Concrete Inputs": ["[82421]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1381170,
    "ts": 1514658021078.800, "dur": 4315.852,
    "args": {
      "External id": 2488936,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[82421], [], [], [], [], [], [], []], "Ev Idx": 5735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1381170,
    "ts": 1514658021082.152, "dur": 4311.420,
    "args": {
      "External id": 2488937,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[82421], [], [], [], [], [], []], "Ev Idx": 5736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658021088.015, "dur": 15.362,
    "args": {
      "External id": 2488938,"Record function id": 0, "Concrete Inputs": ["[82421]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1381170,
    "ts": 1514658021105.636, "dur": 4282.617,
    "args": {
      "External id": 2488939,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[82421], [82421], []], "Ev Idx": 5738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514658025617.322, "dur": 12.134,
    "args": {
      "External id": 2488940,"Record function id": 0, "Concrete Inputs": ["", "[42076416]", "", "", "", "False"], "Input type": ["float", "ScalarList", "", "", "", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[336611328], [], [], [], [], []], "Ev Idx": 5739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1381170,
    "ts": 1514658025621.267, "dur": 7.644,
    "args": {
      "External id": 2488941,"Record function id": 0, "Concrete Inputs": ["[42076416]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_reduce_scatter_base_", "pid": 1336757, "tid": 1381170,
    "ts": 1514658025658.466, "dur": 450.809,
    "args": {
      "External id": 2488942,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "-1"], "Input type": ["float", "float", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], [], []], "Input Dims": [[42076416], [336611328], [], [], [], []], "Ev Idx": 5741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514658025685.128, "dur": 419.219,
    "args": {
      "External id": 2488943,"Record function id": 0, "Collective name": "_reduce_scatter_base", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 42076416, "Process Group Name": "0", "Input type": ["float", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[336611328], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5742, "In msg nelems": 336611328
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_reduce_scatter_base", "pid": 1336757, "tid": 1381170,
    "ts": 1514658025699.438, "dur": 398.748,
    "args": {
      "External id": 2488944,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[336611328]], "Ev Idx": 5743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026129.981, "dur": 2.800,
    "args": {
      "External id": 2488945,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5744, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026195.519, "dur": 7.058,
    "args": {
      "External id": 2488946,"Record function id": 0, "Concrete Inputs": ["", "[4000, 2048]", "[2048, 1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026249.130, "dur": 1.645,
    "args": {
      "External id": 2488947,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8192000"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026266.885, "dur": 1.191,
    "args": {
      "External id": 2488948,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "8192256"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026285.409, "dur": 2.578,
    "args": {
      "External id": 2488949,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "8716544"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026300.128, "dur": 2.006,
    "args": {
      "External id": 2488950,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "9240832"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026311.760, "dur": 1.436,
    "args": {
      "External id": 2488951,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "9765120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026326.293, "dur": 1.445,
    "args": {
      "External id": 2488952,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "10289408"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026338.140, "dur": 3.310,
    "args": {
      "External id": 2488953,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "10289664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026353.197, "dur": 1.378,
    "args": {
      "External id": 2488954,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "11731456"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026365.766, "dur": 1.308,
    "args": {
      "External id": 2488955,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "13173248"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026378.775, "dur": 1.518,
    "args": {
      "External id": 2488956,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "14615040"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026391.766, "dur": 2.775,
    "args": {
      "External id": 2488957,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "14615296"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026406.085, "dur": 1.664,
    "args": {
      "External id": 2488958,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "15139584"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026418.954, "dur": 1.356,
    "args": {
      "External id": 2488959,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "15663872"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026430.227, "dur": 1.357,
    "args": {
      "External id": 2488960,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "16188160"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026442.256, "dur": 3.315,
    "args": {
      "External id": 2488961,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "16712448"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026456.325, "dur": 1.123,
    "args": {
      "External id": 2488962,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "16712704"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026469.135, "dur": 1.256,
    "args": {
      "External id": 2488963,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "18154496"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026480.785, "dur": 1.453,
    "args": {
      "External id": 2488964,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "19596288"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026492.004, "dur": 3.627,
    "args": {
      "External id": 2488965,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21038080"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026505.758, "dur": 1.119,
    "args": {
      "External id": 2488966,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "21038336"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026517.200, "dur": 1.581,
    "args": {
      "External id": 2488967,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "21562624"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026544.637, "dur": 1.991,
    "args": {
      "External id": 2488968,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "22086912"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026560.909, "dur": 3.300,
    "args": {
      "External id": 2488969,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "22611200"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026575.258, "dur": 1.108,
    "args": {
      "External id": 2488970,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "23135488"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026586.760, "dur": 1.305,
    "args": {
      "External id": 2488971,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "23135744"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026598.324, "dur": 1.482,
    "args": {
      "External id": 2488972,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "24577536"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026610.127, "dur": 3.192,
    "args": {
      "External id": 2488973,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "26019328"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026623.698, "dur": 1.336,
    "args": {
      "External id": 2488974,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27461120"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026635.916, "dur": 1.408,
    "args": {
      "External id": 2488975,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "27461376"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026648.027, "dur": 1.439,
    "args": {
      "External id": 2488976,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "27985664"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026659.431, "dur": 3.026,
    "args": {
      "External id": 2488977,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "28509952"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026673.439, "dur": 1.220,
    "args": {
      "External id": 2488978,"Record function id": 0, "Concrete Inputs": ["", "[256, 2048]", "[2048, 1]", "29034240"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026685.022, "dur": 1.437,
    "args": {
      "External id": 2488979,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "29558528"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026696.820, "dur": 1.199,
    "args": {
      "External id": 2488980,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "29558784"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026709.106, "dur": 3.116,
    "args": {
      "External id": 2488981,"Record function id": 0, "Concrete Inputs": ["", "[704, 2048]", "[2048, 1]", "31000576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026722.985, "dur": 1.176,
    "args": {
      "External id": 2488982,"Record function id": 0, "Concrete Inputs": ["", "[256, 5632]", "[5632, 1]", "32442368"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026734.492, "dur": 1.421,
    "args": {
      "External id": 2488983,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "33884160"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1381170,
    "ts": 1514658026746.188, "dur": 1.033,
    "args": {
      "External id": 2488984,"Record function id": 0, "Concrete Inputs": ["", "[4000, 2048]", "[2048, 1]", "33884416"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5783
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "ProfilerStep#102911", "pid": 1336757, "tid": 1336757,
    "ts": 1514655936943.768, "dur": 2108125.280,
    "args": {
      "External id": 2478081,"Record function id": 0, "Ev Idx": 5784
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "Optimizer.zero_grad#AdamW.zero_grad", "pid": 1336757, "tid": 1336757,
    "ts": 1514655936976.143, "dur": 790.448,
    "args": {
      "External id": 2478082,"Record function id": 0, "Ev Idx": 5785
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "enumerate(DataLoader)#_StatefulMultiProcessingDataLoaderIter.__next__", "pid": 1336757, "tid": 1336757,
    "ts": 1514655937808.452, "dur": 2141.031,
    "args": {
      "External id": 2478083,"Record function id": 0, "Ev Idx": 5786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655938911.989, "dur": 7.424,
    "args": {
      "External id": 2478084,"Record function id": 0, "Concrete Inputs": ["[0]", "4", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::set_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655938940.209, "dur": 6.656,
    "args": {
      "External id": 2478085,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "0", "[16, 8192]", "[8192, 1]"], "Input type": ["long int", "", "Scalar", "ScalarList", "ScalarList"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[0], [], [], [], []], "Ev Idx": 5788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655939384.718, "dur": 2.190,
    "args": {
      "External id": 2478086,"Record function id": 0, "Concrete Inputs": ["[0]", "4", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::set_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655939394.021, "dur": 2.326,
    "args": {
      "External id": 2478087,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "0", "[16, 8192]", "[8192, 1]"], "Input type": ["long int", "", "Scalar", "ScalarList", "ScalarList"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[0], [], [], [], []], "Ev Idx": 5790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655939831.406, "dur": 1.690,
    "args": {
      "External id": 2478088,"Record function id": 0, "Concrete Inputs": ["[0]", "4", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::set_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655939838.752, "dur": 2.202,
    "args": {
      "External id": 2478089,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "0", "[16, 8192]", "[8192, 1]"], "Input type": ["long int", "", "Scalar", "ScalarList", "ScalarList"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[0], [], [], [], []], "Ev Idx": 5792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940405.652, "dur": 13.171,
    "args": {
      "External id": 2478090,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], []], "Ev Idx": 5793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940412.658, "dur": 2.446,
    "args": {
      "External id": 2478091,"Record function id": 0, "Concrete Inputs": ["", "[16, 8192]", "[8192, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 5794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940420.343, "dur": 4.355,
    "args": {
      "External id": 2478092,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "4096", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], []], "Ev Idx": 5795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940422.306, "dur": 1.183,
    "args": {
      "External id": 2478093,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 5796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940451.106, "dur": 585.290,
    "args": {
      "External id": 2478094,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], [], [], []], "Ev Idx": 5797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940458.192, "dur": 576.910,
    "args": {
      "External id": 2478095,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], [], []], "Ev Idx": 5798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940466.371, "dur": 10.185,
    "args": {
      "External id": 2478096,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "[4096, 1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940478.701, "dur": 554.453,
    "args": {
      "External id": 2478097,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940488.299, "dur": 0.557,
    "args": {
      "External id": 2478098,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 5801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940491.599, "dur": 8.254,
    "args": {
      "External id": 2478099,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["long int", "long int"], "Input Strides": [[8192, 1], [4096, 1]], "Input Dims": [[16, 4096], [16, 4096]], "Ev Idx": 5802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940496.262, "dur": 3.441,
    "args": {
      "External id": 2478100,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "False"], "Input type": ["long int", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], []], "Input Dims": [[16, 4096], [], []], "Ev Idx": 5803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940498.826, "dur": 0.598,
    "args": {
      "External id": 2478101,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 5804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940502.177, "dur": 251.046,
    "args": {
      "External id": 2478102,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 5805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940504.772, "dur": 248.052,
    "args": {
      "External id": 2478103,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 5806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940506.721, "dur": 13.418,
    "args": {
      "External id": 2478104,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 5807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940508.783, "dur": 10.798,
    "args": {
      "External id": 2478105,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940520.925, "dur": 230.466,
    "args": {
      "External id": 2478106,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655940756.293, "dur": 272.011,
    "args": {
      "External id": 2478107,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [4096, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941057.417, "dur": 770.102,
    "args": {
      "External id": 2478108,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], [], [], [], []], "Ev Idx": 5811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941059.303, "dur": 767.232,
    "args": {
      "External id": 2478109,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], [], [], []], "Ev Idx": 5812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941066.287, "dur": 11.115,
    "args": {
      "External id": 2478110,"Record function id": 0, "Concrete Inputs": ["[16, 8192]", "[8192, 1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941078.528, "dur": 743.830,
    "args": {
      "External id": 2478111,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[8192, 1], [8192, 1], []], "Input Dims": [[16, 8192], [16, 8192], []], "Ev Idx": 5814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::arange", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941858.193, "dur": 55.539,
    "args": {
      "External id": 2478112,"Record function id": 0, "Concrete Inputs": ["0", "4096", "", "", "", "False"], "Input type": ["Scalar", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941864.246, "dur": 5.960,
    "args": {
      "External id": 2478113,"Record function id": 0, "Concrete Inputs": ["[0]", "4", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::arange", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941873.058, "dur": 40.360,
    "args": {
      "External id": 2478114,"Record function id": 0, "Concrete Inputs": ["0", "4096", "1", ""], "Input type": ["Scalar", "Scalar", "Scalar", "long int"], "Input Strides": [[], [], [], [1]], "Input Dims": [[], [], [], [0]], "Ev Idx": 5817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941877.738, "dur": 8.527,
    "args": {
      "External id": 2478115,"Record function id": 0, "Concrete Inputs": ["", "[4096]", ""], "Input type": ["long int", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 5818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::repeat", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941927.118, "dur": 112.865,
    "args": {
      "External id": 2478116,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4096], []], "Ev Idx": 5819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941931.456, "dur": 7.037,
    "args": {
      "External id": 2478117,"Record function id": 0, "Concrete Inputs": ["", "[1, 4096]", "False"], "Input type": ["long int", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[4096], [], []], "Ev Idx": 5820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941936.163, "dur": 1.938,
    "args": {
      "External id": 2478118,"Record function id": 0, "Concrete Inputs": ["", "[1, 4096]", "[4096, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[4096], [], [], []], "Ev Idx": 5821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941939.564, "dur": 5.234,
    "args": {
      "External id": 2478119,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941947.645, "dur": 3.634,
    "args": {
      "External id": 2478120,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[4096, 1]], "Input Dims": [[16, 4096]], "Ev Idx": 5823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unfold", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941955.607, "dur": 5.991,
    "args": {
      "External id": 2478121,"Record function id": 0, "Concrete Inputs": ["", "0", "1", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar"], "Input Strides": [[4096, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 5824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941960.555, "dur": 0.832,
    "args": {
      "External id": 2478122,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 1]", "[4096, 1, 4096]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[4096, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 5825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unfold", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941962.569, "dur": 4.952,
    "args": {
      "External id": 2478123,"Record function id": 0, "Concrete Inputs": ["", "1", "4096", "4096"], "Input type": ["long int", "Scalar", "Scalar", "Scalar"], "Input Strides": [[4096, 1, 4096], [], [], []], "Input Dims": [[16, 4096, 1], [], [], []], "Ev Idx": 5826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941966.546, "dur": 0.737,
    "args": {
      "External id": 2478124,"Record function id": 0, "Concrete Inputs": ["", "[16, 1, 1, 4096]", "[4096, 4096, 4096, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[4096, 1, 4096], [], [], []], "Input Dims": [[16, 4096, 1], [], [], []], "Ev Idx": 5827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941969.874, "dur": 3.587,
    "args": {
      "External id": 2478125,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["long int", "long int"], "Input Strides": [[4096, 1], [4096, 4096, 4096, 1]], "Input Dims": [[1, 4096], [16, 1, 1, 4096]], "Ev Idx": 5828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::expand", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941970.897, "dur": 2.448,
    "args": {
      "External id": 2478126,"Record function id": 0, "Concrete Inputs": ["", "[16, 1, 1, 4096]", "False"], "Input type": ["long int", "ScalarList", "Scalar"], "Input Strides": [[4096, 1], [], []], "Input Dims": [[1, 4096], [], []], "Ev Idx": 5829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941972.601, "dur": 0.648,
    "args": {
      "External id": 2478127,"Record function id": 0, "Concrete Inputs": ["", "[16, 1, 1, 4096]", "[0, 4096, 4096, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[4096, 1], [], [], []], "Input Dims": [[1, 4096], [], [], []], "Ev Idx": 5830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655941974.607, "dur": 63.894,
    "args": {
      "External id": 2478128,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 4096, 4096, 1], [0, 4096, 4096, 1], []], "Input Dims": [[16, 1, 1, 4096], [16, 1, 1, 4096], []], "Ev Idx": 5831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942049.804, "dur": 31.441,
    "args": {
      "External id": 2478129,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "3", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[4096, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 5832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942051.426, "dur": 29.603,
    "args": {
      "External id": 2478130,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "3", "", "", "", "False", ""], "Input type": ["long int", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[4096, 1], [], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], [], []], "Ev Idx": 5833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942058.797, "dur": 3.616,
    "args": {
      "External id": 2478131,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "[4096, 1]", "3", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942063.463, "dur": 17.065,
    "args": {
      "External id": 2478132,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["int", "long int", "Scalar"], "Input Strides": [[4096, 1], [4096, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 5835
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::root_pre_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942197.811, "dur": 142.324,
    "args": {
      "External id": 2478133,"Record function id": 0, "Ev Idx": 5836
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::inputs_to_device", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942278.041, "dur": 51.305,
    "args": {
      "External id": 2478134,"Record function id": 0, "Ev Idx": 5837
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942346.871, "dur": 42.011,
    "args": {
      "External id": 2478135,"Record function id": 0, "Ev Idx": 5838
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942398.144, "dur": 9789.432,
    "args": {
      "External id": 2478136,"Record function id": 0, "Ev Idx": 5839
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942407.227, "dur": 1346.104,
    "args": {
      "External id": 2478137,"Record function id": 0, "Ev Idx": 5840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942546.177, "dur": 8.218,
    "args": {
      "External id": 2478138,"Record function id": 0, "Concrete Inputs": ["[42076416]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942574.294, "dur": 115.295,
    "args": {
      "External id": 2478139,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["c10::BFloat16", "", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[42076416], [], []], "Ev Idx": 5842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942579.414, "dur": 1.964,
    "args": {
      "External id": 2478140,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942586.544, "dur": 0.248,
    "args": {
      "External id": 2478141,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "8192000"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942587.810, "dur": 0.323,
    "args": {
      "External id": 2478142,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "8192256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942588.864, "dur": 1.542,
    "args": {
      "External id": 2478143,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "8716544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942594.448, "dur": 0.336,
    "args": {
      "External id": 2478144,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "9240832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942595.441, "dur": 0.380,
    "args": {
      "External id": 2478145,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "9765120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942596.670, "dur": 1.842,
    "args": {
      "External id": 2478146,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "10289408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942601.461, "dur": 0.485,
    "args": {
      "External id": 2478147,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "10289664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942602.654, "dur": 0.469,
    "args": {
      "External id": 2478148,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "11731456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942607.255, "dur": 0.235,
    "args": {
      "External id": 2478149,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "13173248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942608.381, "dur": 0.289,
    "args": {
      "External id": 2478150,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "14615040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942609.268, "dur": 1.575,
    "args": {
      "External id": 2478151,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "14615296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942613.471, "dur": 0.378,
    "args": {
      "External id": 2478152,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "15139584"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942614.517, "dur": 0.144,
    "args": {
      "External id": 2478153,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "15663872"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942615.595, "dur": 2.039,
    "args": {
      "External id": 2478154,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "16188160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942620.305, "dur": 0.163,
    "args": {
      "External id": 2478155,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "16712448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942621.245, "dur": 0.280,
    "args": {
      "External id": 2478156,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "16712704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942625.807, "dur": 0.138,
    "args": {
      "External id": 2478157,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "18154496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942626.698, "dur": 0.148,
    "args": {
      "External id": 2478158,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "19596288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942627.602, "dur": 1.646,
    "args": {
      "External id": 2478159,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "21038080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942634.248, "dur": 0.176,
    "args": {
      "External id": 2478160,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "21038336"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942635.142, "dur": 0.164,
    "args": {
      "External id": 2478161,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "21562624"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942636.072, "dur": 2.386,
    "args": {
      "External id": 2478162,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "22086912"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942641.160, "dur": 0.252,
    "args": {
      "External id": 2478163,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "22611200"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942642.331, "dur": 0.336,
    "args": {
      "External id": 2478164,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "23135488"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942646.716, "dur": 0.278,
    "args": {
      "External id": 2478165,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "23135744"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942647.814, "dur": 0.299,
    "args": {
      "External id": 2478166,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "24577536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942648.932, "dur": 1.173,
    "args": {
      "External id": 2478167,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "26019328"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942652.959, "dur": 0.176,
    "args": {
      "External id": 2478168,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27461120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942653.804, "dur": 0.193,
    "args": {
      "External id": 2478169,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27461376"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942655.111, "dur": 2.344,
    "args": {
      "External id": 2478170,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27985664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942659.811, "dur": 0.436,
    "args": {
      "External id": 2478171,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "28509952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942661.060, "dur": 0.302,
    "args": {
      "External id": 2478172,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "29034240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942665.417, "dur": 0.144,
    "args": {
      "External id": 2478173,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "29558528"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942666.278, "dur": 0.160,
    "args": {
      "External id": 2478174,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29558784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942667.059, "dur": 0.876,
    "args": {
      "External id": 2478175,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "31000576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942670.473, "dur": 0.273,
    "args": {
      "External id": 2478176,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "32442368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942671.360, "dur": 0.262,
    "args": {
      "External id": 2478177,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "33884160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942672.210, "dur": 2.116,
    "args": {
      "External id": 2478178,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "33884416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942712.053, "dur": 78.388,
    "args": {
      "External id": 2478179,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 5882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942869.841, "dur": 303.540,
    "args": {
      "External id": 2478180,"Record function id": 0, "Concrete Inputs": ["", "", "42076416", "8", "4", "15", ""], "Input type": ["TensorList", "", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], []], "Ev Idx": 5883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942888.337, "dur": 4.435,
    "args": {
      "External id": 2478181,"Record function id": 0, "Concrete Inputs": ["[336611328]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942899.043, "dur": 13.126,
    "args": {
      "External id": 2478182,"Record function id": 0, "Concrete Inputs": ["", "0", "168305664", "42076416"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[336611328], [], [], []], "Ev Idx": 5885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942903.610, "dur": 8.053,
    "args": {
      "External id": 2478183,"Record function id": 0, "Concrete Inputs": ["", "0", "168305664", "210382080", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[336611328], [], [], [], []], "Ev Idx": 5886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942908.522, "dur": 0.633,
    "args": {
      "External id": 2478184,"Record function id": 0, "Concrete Inputs": ["", "[42076416]", "[1]", "168305664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[336611328], [], [], []], "Ev Idx": 5887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942919.824, "dur": 154.427,
    "args": {
      "External id": 2478185,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["c10::BFloat16", "", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[42076416], [], []], "Ev Idx": 5888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942921.663, "dur": 0.392,
    "args": {
      "External id": 2478186,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "168305664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942923.221, "dur": 0.412,
    "args": {
      "External id": 2478187,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "176497664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942926.121, "dur": 1.900,
    "args": {
      "External id": 2478188,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "176497920"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942928.747, "dur": 0.468,
    "args": {
      "External id": 2478189,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "177022208"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942931.505, "dur": 0.322,
    "args": {
      "External id": 2478190,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "177546496"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942932.542, "dur": 0.235,
    "args": {
      "External id": 2478191,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "178070784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942933.956, "dur": 0.314,
    "args": {
      "External id": 2478192,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "178595072"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942937.173, "dur": 0.302,
    "args": {
      "External id": 2478193,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "178595328"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942938.289, "dur": 0.696,
    "args": {
      "External id": 2478194,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "180037120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942939.645, "dur": 0.414,
    "args": {
      "External id": 2478195,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "181478912"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942942.350, "dur": 1.370,
    "args": {
      "External id": 2478196,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "182920704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942944.447, "dur": 0.395,
    "args": {
      "External id": 2478197,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "182920960"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942947.204, "dur": 1.942,
    "args": {
      "External id": 2478198,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "183445248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942949.864, "dur": 0.178,
    "args": {
      "External id": 2478199,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "183969536"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942950.736, "dur": 0.465,
    "args": {
      "External id": 2478200,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "184493824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942954.680, "dur": 0.140,
    "args": {
      "External id": 2478201,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "185018112"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942955.487, "dur": 0.296,
    "args": {
      "External id": 2478202,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "185018368"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942956.818, "dur": 0.260,
    "args": {
      "External id": 2478203,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "186460160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942962.060, "dur": 1.415,
    "args": {
      "External id": 2478204,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "187901952"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942964.128, "dur": 0.150,
    "args": {
      "External id": 2478205,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "189343744"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942967.001, "dur": 2.191,
    "args": {
      "External id": 2478206,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "189344000"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942969.919, "dur": 0.153,
    "args": {
      "External id": 2478207,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "189868288"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942970.869, "dur": 0.352,
    "args": {
      "External id": 2478208,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "190392576"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942975.396, "dur": 0.165,
    "args": {
      "External id": 2478209,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "190916864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942976.410, "dur": 0.266,
    "args": {
      "External id": 2478210,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "191441152"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942977.280, "dur": 0.232,
    "args": {
      "External id": 2478211,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "191441408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655942982.221, "dur": 1.496,
    "args": {
      "External id": 2478212,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "192883200"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943033.039, "dur": 1.052,
    "args": {
      "External id": 2478213,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "194324992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943038.018, "dur": 2.148,
    "args": {
      "External id": 2478214,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "195766784"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943040.889, "dur": 0.271,
    "args": {
      "External id": 2478215,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "195767040"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943041.665, "dur": 0.344,
    "args": {
      "External id": 2478216,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "196291328"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943045.840, "dur": 0.402,
    "args": {
      "External id": 2478217,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "196815616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943047.054, "dur": 0.453,
    "args": {
      "External id": 2478218,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "197339904"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943048.235, "dur": 0.447,
    "args": {
      "External id": 2478219,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "197864192"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943052.668, "dur": 1.040,
    "args": {
      "External id": 2478220,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "197864448"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943054.361, "dur": 0.161,
    "args": {
      "External id": 2478221,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "199306240"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943057.027, "dur": 2.161,
    "args": {
      "External id": 2478222,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "200748032"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943059.818, "dur": 0.271,
    "args": {
      "External id": 2478223,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "202189824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943061.102, "dur": 0.293,
    "args": {
      "External id": 2478224,"Record function id": 0, "Concrete Inputs": ["", "[8192000]", "[1]", "202190080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[42076416], [], [], []], "Ev Idx": 5927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943098.081, "dur": 59.843,
    "args": {
      "External id": 2478225,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 5928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943234.288, "dur": 410.671,
    "args": {
      "External id": 2478226,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[336611328], [42076416], [], [], []], "Ev Idx": 5929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943269.429, "dur": 369.818,
    "args": {
      "External id": 2478227,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 336611328, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[42076416], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 5930, "In msg nelems": 42076416
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943280.423, "dur": 352.834,
    "args": {
      "External id": 2478228,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[42076416]], "Ev Idx": 5931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943670.640, "dur": 2.397,
    "args": {
      "External id": 2478229,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 5932, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out", "pid": 1336757, "tid": 1336757,
    "ts": 1514655943769.577, "dur": 8194.702,
    "args": {
      "External id": 2478230,"Record function id": 0, "Ev Idx": 5933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944029.763, "dur": 7.430,
    "args": {
      "External id": 2478231,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[336611328], []], "Ev Idx": 5934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944041.398, "dur": 0.707,
    "args": {
      "External id": 2478232,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[65536000], []], "Ev Idx": 5935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944043.860, "dur": 1.015,
    "args": {
      "External id": 2478233,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944046.669, "dur": 2.018,
    "args": {
      "External id": 2478234,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944049.960, "dur": 0.889,
    "args": {
      "External id": 2478235,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944054.030, "dur": 0.835,
    "args": {
      "External id": 2478236,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944056.394, "dur": 0.655,
    "args": {
      "External id": 2478237,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944058.229, "dur": 1.720,
    "args": {
      "External id": 2478238,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944063.300, "dur": 0.793,
    "args": {
      "External id": 2478239,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944066.578, "dur": 0.792,
    "args": {
      "External id": 2478240,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944068.858, "dur": 0.659,
    "args": {
      "External id": 2478241,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944070.798, "dur": 1.673,
    "args": {
      "External id": 2478242,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944073.560, "dur": 0.884,
    "args": {
      "External id": 2478243,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944077.729, "dur": 0.931,
    "args": {
      "External id": 2478244,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944079.951, "dur": 0.950,
    "args": {
      "External id": 2478245,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944082.310, "dur": 1.532,
    "args": {
      "External id": 2478246,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944086.627, "dur": 0.850,
    "args": {
      "External id": 2478247,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944090.520, "dur": 0.530,
    "args": {
      "External id": 2478248,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944092.706, "dur": 0.687,
    "args": {
      "External id": 2478249,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944094.771, "dur": 1.860,
    "args": {
      "External id": 2478250,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944097.857, "dur": 0.813,
    "args": {
      "External id": 2478251,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944101.326, "dur": 0.788,
    "args": {
      "External id": 2478252,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944103.449, "dur": 0.760,
    "args": {
      "External id": 2478253,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944105.501, "dur": 1.692,
    "args": {
      "External id": 2478254,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944123.139, "dur": 0.565,
    "args": {
      "External id": 2478255,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944127.213, "dur": 0.715,
    "args": {
      "External id": 2478256,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944129.379, "dur": 0.759,
    "args": {
      "External id": 2478257,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944131.582, "dur": 1.498,
    "args": {
      "External id": 2478258,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944134.251, "dur": 0.950,
    "args": {
      "External id": 2478259,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944138.487, "dur": 0.753,
    "args": {
      "External id": 2478260,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944140.773, "dur": 0.704,
    "args": {
      "External id": 2478261,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944142.827, "dur": 1.640,
    "args": {
      "External id": 2478262,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944147.303, "dur": 0.553,
    "args": {
      "External id": 2478263,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944150.469, "dur": 0.766,
    "args": {
      "External id": 2478264,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 5967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944152.509, "dur": 1.091,
    "args": {
      "External id": 2478265,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944154.702, "dur": 1.800,
    "args": {
      "External id": 2478266,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944157.694, "dur": 0.766,
    "args": {
      "External id": 2478267,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944161.467, "dur": 0.583,
    "args": {
      "External id": 2478268,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 5971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944163.208, "dur": 0.939,
    "args": {
      "External id": 2478269,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 5972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944165.534, "dur": 1.548,
    "args": {
      "External id": 2478270,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[65536000], []], "Ev Idx": 5973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944195.662, "dur": 7716.687,
    "args": {
      "External id": 2478271,"Record function id": 0, "Concrete Inputs": ["", "", "1", ""], "Input type": ["c10::BFloat16", "", "Scalar", "TensorList"], "Input Strides": [[42076416, 1], [], [], []], "Input Dims": [[8, 42076416], [], [], []], "Ev Idx": 5974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944221.047, "dur": 7682.070,
    "args": {
      "External id": 2478272,"Record function id": 0, "Concrete Inputs": ["", "", "1", ""], "Input type": ["c10::BFloat16", "", "Scalar", "TensorList"], "Input Strides": [[42076416, 1], [], [], []], "Input Dims": [[8, 42076416], [], [], []], "Ev Idx": 5975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944242.476, "dur": 5.842,
    "args": {
      "External id": 2478273,"Record function id": 0, "Concrete Inputs": ["[4290]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944255.159, "dur": 7610.170,
    "args": {
      "External id": 2478274,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[4290], [], [], [], [], [], [], []], "Ev Idx": 5977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944257.594, "dur": 7607.083,
    "args": {
      "External id": 2478275,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[4290], [], [], [], [], [], []], "Ev Idx": 5978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944264.158, "dur": 5.795,
    "args": {
      "External id": 2478276,"Record function id": 0, "Concrete Inputs": ["[4290]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655944271.676, "dur": 7589.732,
    "args": {
      "External id": 2478277,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[4290], [4290], []], "Ev Idx": 5980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952268.446, "dur": 32.604,
    "args": {
      "External id": 2478278,"Record function id": 0, "Ev Idx": 5981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 0/0", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952302.141, "dur": 210.103,
    "args": {
      "External id": 2478279,"Record function id": 0, "Ev Idx": 5982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952343.655, "dur": 159.816,
    "args": {
      "External id": 2478280,"Sequence number": 24740420, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "long int"], "Input Strides": [[2048, 1], [4096, 1]], "Input Dims": [[32000, 2048], [16, 4096]], "Ev Idx": 5983
    }
  },
  {
    "ph": "s", "id": 228, "pid": 1336757, "tid": 1336757, "ts": 1514655952343.655,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_embedding_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952418.582, "dur": 48.859,
    "args": {
      "External id": 2478281,"kernel_hash": "c5avyhm3orrzkqjkvrj7bppgyoz2vz6xunjvoc7l6ifi26al6auf", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/5a/c5avyhm3orrzkqjkvrj7bppgyoz2vz6xunjvoc7l6ifi26al6auf.py", "kernel_backend": "triton", "Input type": ["long int", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[4096, 1], [2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096], [32000, 2048], [16, 4096, 2048], []], "Ev Idx": 5984
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952592.126, "dur": 51.398,
    "args": {
      "External id": 2478282,"Record function id": 0, "Ev Idx": 5985
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.0)", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952654.220, "dur": 8239.779,
    "args": {
      "External id": 2478283,"Record function id": 0, "Ev Idx": 5986
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.0)", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952662.046, "dur": 794.445,
    "args": {
      "External id": 2478284,"Record function id": 0, "Ev Idx": 5987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952737.762, "dur": 9.806,
    "args": {
      "External id": 2478285,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 5988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952760.905, "dur": 37.241,
    "args": {
      "External id": 2478286,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 5989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952769.304, "dur": 2.438,
    "args": {
      "External id": 2478287,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952773.404, "dur": 0.592,
    "args": {
      "External id": 2478288,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952774.740, "dur": 1.795,
    "args": {
      "External id": 2478289,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952779.271, "dur": 0.365,
    "args": {
      "External id": 2478290,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952780.364, "dur": 0.368,
    "args": {
      "External id": 2478291,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952784.722, "dur": 0.215,
    "args": {
      "External id": 2478292,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952785.621, "dur": 0.208,
    "args": {
      "External id": 2478293,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952786.580, "dur": 1.402,
    "args": {
      "External id": 2478294,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952792.218, "dur": 0.173,
    "args": {
      "External id": 2478295,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 5998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952808.025, "dur": 35.733,
    "args": {
      "External id": 2478296,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 5999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952874.509, "dur": 102.989,
    "args": {
      "External id": 2478297,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952885.233, "dur": 3.527,
    "args": {
      "External id": 2478298,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952893.826, "dur": 11.721,
    "args": {
      "External id": 2478299,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952898.614, "dur": 6.525,
    "args": {
      "External id": 2478300,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952903.458, "dur": 0.586,
    "args": {
      "External id": 2478301,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952911.620, "dur": 26.339,
    "args": {
      "External id": 2478302,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952913.018, "dur": 0.279,
    "args": {
      "External id": 2478303,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952916.277, "dur": 0.531,
    "args": {
      "External id": 2478304,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952919.097, "dur": 0.541,
    "args": {
      "External id": 2478305,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952921.844, "dur": 0.578,
    "args": {
      "External id": 2478306,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952923.070, "dur": 3.082,
    "args": {
      "External id": 2478307,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952926.981, "dur": 0.569,
    "args": {
      "External id": 2478308,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952929.948, "dur": 0.424,
    "args": {
      "External id": 2478309,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952933.370, "dur": 0.286,
    "args": {
      "External id": 2478310,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952934.317, "dur": 0.169,
    "args": {
      "External id": 2478311,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655952948.590, "dur": 20.928,
    "args": {
      "External id": 2478312,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953092.211, "dur": 282.611,
    "args": {
      "External id": 2478313,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953120.853, "dur": 249.552,
    "args": {
      "External id": 2478314,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6017, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953130.732, "dur": 234.603,
    "args": {
      "External id": 2478315,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953393.698, "dur": 2.100,
    "args": {
      "External id": 2478316,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6019, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.0)", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953474.693, "dur": 7112.301,
    "args": {
      "External id": 2478317,"Record function id": 0, "Ev Idx": 6020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953592.119, "dur": 6.513,
    "args": {
      "External id": 2478318,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953602.530, "dur": 1.000,
    "args": {
      "External id": 2478319,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953605.433, "dur": 0.770,
    "args": {
      "External id": 2478320,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953609.449, "dur": 2.766,
    "args": {
      "External id": 2478321,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953613.642, "dur": 1.081,
    "args": {
      "External id": 2478322,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953616.150, "dur": 0.775,
    "args": {
      "External id": 2478323,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953618.750, "dur": 0.968,
    "args": {
      "External id": 2478324,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953624.136, "dur": 2.076,
    "args": {
      "External id": 2478325,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953627.759, "dur": 0.724,
    "args": {
      "External id": 2478326,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953629.663, "dur": 0.770,
    "args": {
      "External id": 2478327,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953647.582, "dur": 6831.022,
    "args": {
      "External id": 2478328,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953664.000, "dur": 6799.881,
    "args": {
      "External id": 2478329,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953689.275, "dur": 13.324,
    "args": {
      "External id": 2478330,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953708.245, "dur": 6703.210,
    "args": {
      "External id": 2478331,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953710.821, "dur": 6699.520,
    "args": {
      "External id": 2478332,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953716.189, "dur": 7.445,
    "args": {
      "External id": 2478333,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655953725.291, "dur": 6679.158,
    "args": {
      "External id": 2478334,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514655960813.176, "dur": 45.498,
    "args": {
      "External id": 2478335,"Sequence number": 24740421, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6038
    }
  },
  {
    "ph": "s", "id": 227, "pid": 1336757, "tid": 1336757, "ts": 1514655960813.176,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514655960837.313, "dur": 15.491,
    "args": {
      "External id": 2478336,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655960844.873, "dur": 7.455,
    "args": {
      "External id": 2478337,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514655960948.281, "dur": 180.607,
    "args": {
      "External id": 2478338,"Record function id": 0, "Ev Idx": 6041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514655961133.787, "dur": 1694.170,
    "args": {
      "External id": 2478339,"Record function id": 0, "Ev Idx": 6042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514655961192.523, "dur": 1615.729,
    "args": {
      "External id": 2478340,"Sequence number": 24740422, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6043
    }
  },
  {
    "ph": "s", "id": 226, "pid": 1336757, "tid": 1336757, "ts": 1514655961192.523,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514655961288.813, "dur": 64.861,
    "args": {
      "External id": 2478341,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514655961373.643, "dur": 120.216,
    "args": {
      "External id": 2478342,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514655961510.640, "dur": 89.831,
    "args": {
      "External id": 2478343,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514655961620.259, "dur": 49.522,
    "args": {
      "External id": 2478344,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514655961711.134, "dur": 42.153,
    "args": {
      "External id": 2478345,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514655961780.870, "dur": 21.974,
    "args": {
      "External id": 2478346,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514655961831.904, "dur": 246.428,
    "args": {
      "External id": 2478347,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514655961908.916, "dur": 14.661,
    "args": {
      "External id": 2478348,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655961915.050, "dur": 7.362,
    "args": {
      "External id": 2478349,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655961927.035, "dur": 4.983,
    "args": {
      "External id": 2478350,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655961935.653, "dur": 1.417,
    "args": {
      "External id": 2478351,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655961940.537, "dur": 5.483,
    "args": {
      "External id": 2478352,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514655962098.039, "dur": 86.165,
    "args": {
      "External id": 2478353,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514655962236.302, "dur": 50.968,
    "args": {
      "External id": 2478354,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514655962299.944, "dur": 70.632,
    "args": {
      "External id": 2478355,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514655962389.183, "dur": 59.057,
    "args": {
      "External id": 2478356,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514655962481.456, "dur": 41.303,
    "args": {
      "External id": 2478357,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514655962563.290, "dur": 62.249,
    "args": {
      "External id": 2478358,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514655962664.637, "dur": 33.913,
    "args": {
      "External id": 2478359,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6062
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.0)", "pid": 1336757, "tid": 1336757,
    "ts": 1514655962911.073, "dur": 183.164,
    "args": {
      "External id": 2478360,"Record function id": 0, "Ev Idx": 6063
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963194.560, "dur": 62.133,
    "args": {
      "External id": 2478361,"Record function id": 0, "Ev Idx": 6064
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.1)", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963269.720, "dur": 17224.122,
    "args": {
      "External id": 2478362,"Record function id": 0, "Ev Idx": 6065
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.1)", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963283.083, "dur": 1327.390,
    "args": {
      "External id": 2478363,"Record function id": 0, "Ev Idx": 6066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963387.721, "dur": 12.409,
    "args": {
      "External id": 2478364,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963418.526, "dur": 50.707,
    "args": {
      "External id": 2478365,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963426.428, "dur": 5.326,
    "args": {
      "External id": 2478366,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963434.004, "dur": 0.617,
    "args": {
      "External id": 2478367,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963437.425, "dur": 0.484,
    "args": {
      "External id": 2478368,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963441.699, "dur": 0.768,
    "args": {
      "External id": 2478369,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963443.936, "dur": 1.956,
    "args": {
      "External id": 2478370,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963449.428, "dur": 0.407,
    "args": {
      "External id": 2478371,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963453.083, "dur": 0.426,
    "args": {
      "External id": 2478372,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963454.973, "dur": 0.399,
    "args": {
      "External id": 2478373,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963458.525, "dur": 2.735,
    "args": {
      "External id": 2478374,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963483.235, "dur": 94.761,
    "args": {
      "External id": 2478375,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963632.639, "dur": 166.610,
    "args": {
      "External id": 2478376,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963651.033, "dur": 8.091,
    "args": {
      "External id": 2478377,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963666.581, "dur": 13.921,
    "args": {
      "External id": 2478378,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963672.143, "dur": 7.835,
    "args": {
      "External id": 2478379,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963676.385, "dur": 1.192,
    "args": {
      "External id": 2478380,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963691.335, "dur": 39.894,
    "args": {
      "External id": 2478381,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963695.683, "dur": 0.708,
    "args": {
      "External id": 2478382,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963698.086, "dur": 1.875,
    "args": {
      "External id": 2478383,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963701.226, "dur": 2.375,
    "args": {
      "External id": 2478384,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963707.288, "dur": 0.639,
    "args": {
      "External id": 2478385,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963709.821, "dur": 0.359,
    "args": {
      "External id": 2478386,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963713.226, "dur": 0.485,
    "args": {
      "External id": 2478387,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963717.276, "dur": 0.403,
    "args": {
      "External id": 2478388,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963719.006, "dur": 0.401,
    "args": {
      "External id": 2478389,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963724.644, "dur": 0.466,
    "args": {
      "External id": 2478390,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963749.350, "dur": 36.344,
    "args": {
      "External id": 2478391,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963874.646, "dur": 568.266,
    "args": {
      "External id": 2478392,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963912.286, "dur": 523.211,
    "args": {
      "External id": 2478393,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6096, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514655963925.796, "dur": 499.930,
    "args": {
      "External id": 2478394,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964474.717, "dur": 3.783,
    "args": {
      "External id": 2478395,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6098, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.1)", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964642.053, "dur": 15624.869,
    "args": {
      "External id": 2478396,"Record function id": 0, "Ev Idx": 6099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964781.161, "dur": 7.758,
    "args": {
      "External id": 2478397,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964793.824, "dur": 1.510,
    "args": {
      "External id": 2478398,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964798.044, "dur": 1.479,
    "args": {
      "External id": 2478399,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964803.021, "dur": 1.428,
    "args": {
      "External id": 2478400,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964808.002, "dur": 1.383,
    "args": {
      "External id": 2478401,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964813.840, "dur": 1.337,
    "args": {
      "External id": 2478402,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964820.053, "dur": 1.528,
    "args": {
      "External id": 2478403,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964824.056, "dur": 3.743,
    "args": {
      "External id": 2478404,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964830.776, "dur": 1.062,
    "args": {
      "External id": 2478405,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964836.411, "dur": 1.054,
    "args": {
      "External id": 2478406,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964863.274, "dur": 15333.720,
    "args": {
      "External id": 2478407,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964886.414, "dur": 15297.602,
    "args": {
      "External id": 2478408,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964921.246, "dur": 17.471,
    "args": {
      "External id": 2478409,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964943.598, "dur": 15192.955,
    "args": {
      "External id": 2478410,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964946.723, "dur": 15188.330,
    "args": {
      "External id": 2478411,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964953.093, "dur": 6.696,
    "args": {
      "External id": 2478412,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655964961.726, "dur": 15166.886,
    "args": {
      "External id": 2478413,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514655980431.685, "dur": 36.050,
    "args": {
      "External id": 2478414,"Sequence number": 24740423, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6117
    }
  },
  {
    "ph": "s", "id": 225, "pid": 1336757, "tid": 1336757, "ts": 1514655980431.685,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514655980449.346, "dur": 13.563,
    "args": {
      "External id": 2478415,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655980456.123, "dur": 6.480,
    "args": {
      "External id": 2478416,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514655980548.514, "dur": 83.572,
    "args": {
      "External id": 2478417,"Record function id": 0, "Ev Idx": 6120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514655980634.235, "dur": 1100.017,
    "args": {
      "External id": 2478418,"Record function id": 0, "Ev Idx": 6121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514655980677.105, "dur": 1044.025,
    "args": {
      "External id": 2478419,"Sequence number": 24740424, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6122
    }
  },
  {
    "ph": "s", "id": 224, "pid": 1336757, "tid": 1336757, "ts": 1514655980677.105,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514655980761.801, "dur": 46.277,
    "args": {
      "External id": 2478420,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514655980823.410, "dur": 107.723,
    "args": {
      "External id": 2478421,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514655980939.731, "dur": 36.446,
    "args": {
      "External id": 2478422,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981016.664, "dur": 37.148,
    "args": {
      "External id": 2478423,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981089.539, "dur": 27.701,
    "args": {
      "External id": 2478424,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981132.267, "dur": 17.538,
    "args": {
      "External id": 2478425,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981166.771, "dur": 134.224,
    "args": {
      "External id": 2478426,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981219.068, "dur": 13.281,
    "args": {
      "External id": 2478427,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981224.261, "dur": 7.306,
    "args": {
      "External id": 2478428,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981236.558, "dur": 4.987,
    "args": {
      "External id": 2478429,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981242.801, "dur": 2.553,
    "args": {
      "External id": 2478430,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981247.457, "dur": 2.905,
    "args": {
      "External id": 2478431,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981312.424, "dur": 45.958,
    "args": {
      "External id": 2478432,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981389.757, "dur": 26.967,
    "args": {
      "External id": 2478433,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981425.688, "dur": 39.859,
    "args": {
      "External id": 2478434,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981471.647, "dur": 34.472,
    "args": {
      "External id": 2478435,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981547.843, "dur": 29.068,
    "args": {
      "External id": 2478436,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981584.310, "dur": 37.531,
    "args": {
      "External id": 2478437,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981640.046, "dur": 18.772,
    "args": {
      "External id": 2478438,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6141
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.1)", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981798.434, "dur": 82.751,
    "args": {
      "External id": 2478439,"Record function id": 0, "Ev Idx": 6142
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514655981955.406, "dur": 88.008,
    "args": {
      "External id": 2478440,"Record function id": 0, "Ev Idx": 6143
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.2)", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982054.337, "dur": 18132.377,
    "args": {
      "External id": 2478441,"Record function id": 0, "Ev Idx": 6144
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.2)", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982066.031, "dur": 878.472,
    "args": {
      "External id": 2478442,"Record function id": 0, "Ev Idx": 6145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982151.031, "dur": 9.597,
    "args": {
      "External id": 2478443,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982174.943, "dur": 39.002,
    "args": {
      "External id": 2478444,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982179.948, "dur": 2.490,
    "args": {
      "External id": 2478445,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982186.323, "dur": 0.370,
    "args": {
      "External id": 2478446,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982189.752, "dur": 0.500,
    "args": {
      "External id": 2478447,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982191.404, "dur": 0.297,
    "args": {
      "External id": 2478448,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982195.080, "dur": 0.329,
    "args": {
      "External id": 2478449,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982197.934, "dur": 0.444,
    "args": {
      "External id": 2478450,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982199.288, "dur": 3.226,
    "args": {
      "External id": 2478451,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982205.394, "dur": 0.413,
    "args": {
      "External id": 2478452,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982206.393, "dur": 0.494,
    "args": {
      "External id": 2478453,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982224.962, "dur": 41.753,
    "args": {
      "External id": 2478454,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982301.649, "dur": 107.943,
    "args": {
      "External id": 2478455,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982313.928, "dur": 3.763,
    "args": {
      "External id": 2478456,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982322.897, "dur": 9.907,
    "args": {
      "External id": 2478457,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982327.374, "dur": 5.001,
    "args": {
      "External id": 2478458,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982330.457, "dur": 0.630,
    "args": {
      "External id": 2478459,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982338.740, "dur": 31.533,
    "args": {
      "External id": 2478460,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982342.020, "dur": 2.100,
    "args": {
      "External id": 2478461,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982344.965, "dur": 0.529,
    "args": {
      "External id": 2478462,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982346.400, "dur": 0.244,
    "args": {
      "External id": 2478463,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982351.633, "dur": 1.161,
    "args": {
      "External id": 2478464,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982353.515, "dur": 0.157,
    "args": {
      "External id": 2478465,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982356.275, "dur": 0.418,
    "args": {
      "External id": 2478466,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982358.882, "dur": 0.288,
    "args": {
      "External id": 2478467,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982359.964, "dur": 0.320,
    "args": {
      "External id": 2478468,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982362.895, "dur": 1.920,
    "args": {
      "External id": 2478469,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982380.576, "dur": 21.389,
    "args": {
      "External id": 2478470,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982462.887, "dur": 384.881,
    "args": {
      "External id": 2478471,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982494.272, "dur": 348.147,
    "args": {
      "External id": 2478472,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6175, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982505.684, "dur": 330.117,
    "args": {
      "External id": 2478473,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982873.600, "dur": 2.640,
    "args": {
      "External id": 2478474,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6177, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.2)", "pid": 1336757, "tid": 1336757,
    "ts": 1514655982964.473, "dur": 16878.356,
    "args": {
      "External id": 2478475,"Record function id": 0, "Ev Idx": 6178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983107.549, "dur": 6.868,
    "args": {
      "External id": 2478476,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983117.719, "dur": 1.104,
    "args": {
      "External id": 2478477,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983120.545, "dur": 2.517,
    "args": {
      "External id": 2478478,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983124.731, "dur": 1.038,
    "args": {
      "External id": 2478479,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983127.363, "dur": 1.007,
    "args": {
      "External id": 2478480,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983129.706, "dur": 1.072,
    "args": {
      "External id": 2478481,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983134.308, "dur": 0.851,
    "args": {
      "External id": 2478482,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983136.678, "dur": 2.419,
    "args": {
      "External id": 2478483,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983140.413, "dur": 0.714,
    "args": {
      "External id": 2478484,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983142.288, "dur": 0.905,
    "args": {
      "External id": 2478485,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983164.103, "dur": 16603.189,
    "args": {
      "External id": 2478486,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983179.795, "dur": 16572.814,
    "args": {
      "External id": 2478487,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983203.347, "dur": 13.671,
    "args": {
      "External id": 2478488,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983219.912, "dur": 16475.910,
    "args": {
      "External id": 2478489,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983222.634, "dur": 16471.609,
    "args": {
      "External id": 2478490,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983228.076, "dur": 4.736,
    "args": {
      "External id": 2478491,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514655983234.609, "dur": 16452.871,
    "args": {
      "External id": 2478492,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656000104.190, "dur": 43.797,
    "args": {
      "External id": 2478493,"Sequence number": 24740425, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6196
    }
  },
  {
    "ph": "s", "id": 223, "pid": 1336757, "tid": 1336757, "ts": 1514656000104.190,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656000127.848, "dur": 14.038,
    "args": {
      "External id": 2478494,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656000133.424, "dur": 8.003,
    "args": {
      "External id": 2478495,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656000237.845, "dur": 99.739,
    "args": {
      "External id": 2478496,"Record function id": 0, "Ev Idx": 6199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656000339.924, "dur": 1605.568,
    "args": {
      "External id": 2478497,"Record function id": 0, "Ev Idx": 6200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656000390.206, "dur": 1534.024,
    "args": {
      "External id": 2478498,"Sequence number": 24740426, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6201
    }
  },
  {
    "ph": "s", "id": 222, "pid": 1336757, "tid": 1336757, "ts": 1514656000390.206,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656000479.859, "dur": 96.554,
    "args": {
      "External id": 2478499,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656000597.098, "dur": 119.291,
    "args": {
      "External id": 2478500,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656000731.714, "dur": 52.819,
    "args": {
      "External id": 2478501,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656000799.702, "dur": 51.319,
    "args": {
      "External id": 2478502,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656000891.839, "dur": 39.661,
    "args": {
      "External id": 2478503,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656000958.634, "dur": 97.428,
    "args": {
      "External id": 2478504,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656001089.173, "dur": 182.621,
    "args": {
      "External id": 2478505,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656001159.649, "dur": 19.565,
    "args": {
      "External id": 2478506,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656001169.010, "dur": 8.999,
    "args": {
      "External id": 2478507,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656001182.350, "dur": 8.033,
    "args": {
      "External id": 2478508,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656001192.544, "dur": 1.300,
    "args": {
      "External id": 2478509,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656001197.277, "dur": 5.329,
    "args": {
      "External id": 2478510,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656001288.260, "dur": 84.687,
    "args": {
      "External id": 2478511,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656001419.313, "dur": 48.157,
    "args": {
      "External id": 2478512,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656001480.873, "dur": 91.028,
    "args": {
      "External id": 2478513,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656001588.242, "dur": 56.521,
    "args": {
      "External id": 2478514,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656001675.798, "dur": 34.571,
    "args": {
      "External id": 2478515,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656001720.549, "dur": 50.085,
    "args": {
      "External id": 2478516,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656001798.659, "dur": 25.092,
    "args": {
      "External id": 2478517,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6220
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.2)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002091.536, "dur": 105.779,
    "args": {
      "External id": 2478518,"Record function id": 0, "Ev Idx": 6221
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002295.774, "dur": 64.604,
    "args": {
      "External id": 2478519,"Record function id": 0, "Ev Idx": 6222
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.3)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002374.969, "dur": 17428.624,
    "args": {
      "External id": 2478520,"Record function id": 0, "Ev Idx": 6223
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.3)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002388.896, "dur": 1235.146,
    "args": {
      "External id": 2478521,"Record function id": 0, "Ev Idx": 6224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002495.161, "dur": 12.017,
    "args": {
      "External id": 2478522,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002523.683, "dur": 83.251,
    "args": {
      "External id": 2478523,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002557.426, "dur": 2.612,
    "args": {
      "External id": 2478524,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002567.673, "dur": 0.639,
    "args": {
      "External id": 2478525,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002569.883, "dur": 0.798,
    "args": {
      "External id": 2478526,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002575.947, "dur": 0.740,
    "args": {
      "External id": 2478527,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002582.393, "dur": 0.836,
    "args": {
      "External id": 2478528,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002584.549, "dur": 0.498,
    "args": {
      "External id": 2478529,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002588.386, "dur": 4.120,
    "args": {
      "External id": 2478530,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002595.479, "dur": 0.727,
    "args": {
      "External id": 2478531,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002597.804, "dur": 0.609,
    "args": {
      "External id": 2478532,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002625.601, "dur": 58.026,
    "args": {
      "External id": 2478533,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002732.706, "dur": 171.395,
    "args": {
      "External id": 2478534,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002749.900, "dur": 7.012,
    "args": {
      "External id": 2478535,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002763.890, "dur": 13.101,
    "args": {
      "External id": 2478536,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002769.378, "dur": 7.057,
    "args": {
      "External id": 2478537,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002773.633, "dur": 0.974,
    "args": {
      "External id": 2478538,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002789.537, "dur": 40.847,
    "args": {
      "External id": 2478539,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002792.736, "dur": 3.386,
    "args": {
      "External id": 2478540,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002797.681, "dur": 0.564,
    "args": {
      "External id": 2478541,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002802.273, "dur": 0.704,
    "args": {
      "External id": 2478542,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002806.508, "dur": 2.051,
    "args": {
      "External id": 2478543,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002812.409, "dur": 0.614,
    "args": {
      "External id": 2478544,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002814.148, "dur": 0.369,
    "args": {
      "External id": 2478545,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002817.509, "dur": 0.381,
    "args": {
      "External id": 2478546,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002820.445, "dur": 0.390,
    "args": {
      "External id": 2478547,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002822.308, "dur": 2.204,
    "args": {
      "External id": 2478548,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002857.960, "dur": 34.459,
    "args": {
      "External id": 2478549,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656002975.787, "dur": 511.189,
    "args": {
      "External id": 2478550,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003066.477, "dur": 414.654,
    "args": {
      "External id": 2478551,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6254, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003082.818, "dur": 392.174,
    "args": {
      "External id": 2478552,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003514.455, "dur": 2.518,
    "args": {
      "External id": 2478553,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6256, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.3)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003647.215, "dur": 15924.876,
    "args": {
      "External id": 2478554,"Record function id": 0, "Ev Idx": 6257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003753.131, "dur": 7.383,
    "args": {
      "External id": 2478555,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003764.227, "dur": 1.146,
    "args": {
      "External id": 2478556,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003766.947, "dur": 3.087,
    "args": {
      "External id": 2478557,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003772.125, "dur": 0.844,
    "args": {
      "External id": 2478558,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003774.118, "dur": 1.384,
    "args": {
      "External id": 2478559,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003776.967, "dur": 0.959,
    "args": {
      "External id": 2478560,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003781.206, "dur": 1.305,
    "args": {
      "External id": 2478561,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003783.921, "dur": 2.608,
    "args": {
      "External id": 2478562,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003787.801, "dur": 0.807,
    "args": {
      "External id": 2478563,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003789.908, "dur": 0.972,
    "args": {
      "External id": 2478564,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003812.205, "dur": 15675.927,
    "args": {
      "External id": 2478565,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003829.072, "dur": 15645.782,
    "args": {
      "External id": 2478566,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003852.349, "dur": 15.206,
    "args": {
      "External id": 2478567,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003870.654, "dur": 15555.745,
    "args": {
      "External id": 2478568,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003873.253, "dur": 15551.821,
    "args": {
      "External id": 2478569,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003880.257, "dur": 6.092,
    "args": {
      "External id": 2478570,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656003888.078, "dur": 15530.598,
    "args": {
      "External id": 2478571,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656019730.336, "dur": 46.520,
    "args": {
      "External id": 2478572,"Sequence number": 24740427, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6275
    }
  },
  {
    "ph": "s", "id": 221, "pid": 1336757, "tid": 1336757, "ts": 1514656019730.336,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656019759.889, "dur": 11.922,
    "args": {
      "External id": 2478573,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656019764.854, "dur": 6.755,
    "args": {
      "External id": 2478574,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656019842.053, "dur": 80.671,
    "args": {
      "External id": 2478575,"Record function id": 0, "Ev Idx": 6278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656019926.196, "dur": 1140.487,
    "args": {
      "External id": 2478576,"Record function id": 0, "Ev Idx": 6279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656019967.312, "dur": 1084.539,
    "args": {
      "External id": 2478577,"Sequence number": 24740428, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6280
    }
  },
  {
    "ph": "s", "id": 220, "pid": 1336757, "tid": 1336757, "ts": 1514656019967.312,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020076.997, "dur": 48.865,
    "args": {
      "External id": 2478578,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020139.399, "dur": 103.743,
    "args": {
      "External id": 2478579,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020255.054, "dur": 37.178,
    "args": {
      "External id": 2478580,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020298.095, "dur": 30.421,
    "args": {
      "External id": 2478581,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020355.463, "dur": 25.237,
    "args": {
      "External id": 2478582,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020396.817, "dur": 14.970,
    "args": {
      "External id": 2478583,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020430.898, "dur": 156.145,
    "args": {
      "External id": 2478584,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020487.929, "dur": 11.933,
    "args": {
      "External id": 2478585,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020493.428, "dur": 5.498,
    "args": {
      "External id": 2478586,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020502.208, "dur": 5.052,
    "args": {
      "External id": 2478587,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020508.712, "dur": 1.266,
    "args": {
      "External id": 2478588,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020514.133, "dur": 4.314,
    "args": {
      "External id": 2478589,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020599.371, "dur": 48.132,
    "args": {
      "External id": 2478590,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020680.665, "dur": 30.009,
    "args": {
      "External id": 2478591,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020719.784, "dur": 40.287,
    "args": {
      "External id": 2478592,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020788.336, "dur": 35.026,
    "args": {
      "External id": 2478593,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020850.528, "dur": 24.601,
    "args": {
      "External id": 2478594,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020880.771, "dur": 33.363,
    "args": {
      "External id": 2478595,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656020930.388, "dur": 19.118,
    "args": {
      "External id": 2478596,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6299
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.3)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021133.365, "dur": 73.210,
    "args": {
      "External id": 2478597,"Record function id": 0, "Ev Idx": 6300
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021278.054, "dur": 44.407,
    "args": {
      "External id": 2478598,"Record function id": 0, "Ev Idx": 6301
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.4)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021331.109, "dur": 17932.056,
    "args": {
      "External id": 2478599,"Record function id": 0, "Ev Idx": 6302
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.4)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021341.837, "dur": 929.756,
    "args": {
      "External id": 2478600,"Record function id": 0, "Ev Idx": 6303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021423.442, "dur": 9.007,
    "args": {
      "External id": 2478601,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021445.054, "dur": 40.922,
    "args": {
      "External id": 2478602,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021451.508, "dur": 2.513,
    "args": {
      "External id": 2478603,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021457.651, "dur": 0.550,
    "args": {
      "External id": 2478604,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021459.138, "dur": 0.300,
    "args": {
      "External id": 2478605,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021462.491, "dur": 0.629,
    "args": {
      "External id": 2478606,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021466.015, "dur": 0.639,
    "args": {
      "External id": 2478607,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021468.966, "dur": 0.639,
    "args": {
      "External id": 2478608,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021470.221, "dur": 3.619,
    "args": {
      "External id": 2478609,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021474.492, "dur": 0.541,
    "args": {
      "External id": 2478610,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021477.941, "dur": 0.412,
    "args": {
      "External id": 2478611,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021496.639, "dur": 60.122,
    "args": {
      "External id": 2478612,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021592.707, "dur": 118.047,
    "args": {
      "External id": 2478613,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021605.716, "dur": 4.824,
    "args": {
      "External id": 2478614,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021615.820, "dur": 13.041,
    "args": {
      "External id": 2478615,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021620.547, "dur": 7.870,
    "args": {
      "External id": 2478616,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021626.037, "dur": 0.899,
    "args": {
      "External id": 2478617,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021635.515, "dur": 31.006,
    "args": {
      "External id": 2478618,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021637.133, "dur": 2.424,
    "args": {
      "External id": 2478619,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021642.122, "dur": 0.463,
    "args": {
      "External id": 2478620,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021643.240, "dur": 0.412,
    "args": {
      "External id": 2478621,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021647.980, "dur": 2.033,
    "args": {
      "External id": 2478622,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021650.814, "dur": 0.340,
    "args": {
      "External id": 2478623,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021651.811, "dur": 0.598,
    "args": {
      "External id": 2478624,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021656.435, "dur": 0.453,
    "args": {
      "External id": 2478625,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021657.579, "dur": 0.232,
    "args": {
      "External id": 2478626,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021658.723, "dur": 2.155,
    "args": {
      "External id": 2478627,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021678.550, "dur": 24.211,
    "args": {
      "External id": 2478628,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021765.616, "dur": 405.370,
    "args": {
      "External id": 2478629,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021796.374, "dur": 369.051,
    "args": {
      "External id": 2478630,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6333, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656021808.689, "dur": 350.564,
    "args": {
      "External id": 2478631,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022196.575, "dur": 2.265,
    "args": {
      "External id": 2478632,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6335, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.4)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022292.646, "dur": 16777.659,
    "args": {
      "External id": 2478633,"Record function id": 0, "Ev Idx": 6336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022394.483, "dur": 6.455,
    "args": {
      "External id": 2478634,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022404.506, "dur": 1.321,
    "args": {
      "External id": 2478635,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022407.676, "dur": 2.778,
    "args": {
      "External id": 2478636,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022412.346, "dur": 1.331,
    "args": {
      "External id": 2478637,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022415.201, "dur": 1.039,
    "args": {
      "External id": 2478638,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022417.518, "dur": 0.878,
    "args": {
      "External id": 2478639,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022421.808, "dur": 1.093,
    "args": {
      "External id": 2478640,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022425.486, "dur": 2.771,
    "args": {
      "External id": 2478641,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022429.779, "dur": 0.899,
    "args": {
      "External id": 2478642,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022432.188, "dur": 0.811,
    "args": {
      "External id": 2478643,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022452.672, "dur": 16574.492,
    "args": {
      "External id": 2478644,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022469.907, "dur": 16549.866,
    "args": {
      "External id": 2478645,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022493.810, "dur": 15.224,
    "args": {
      "External id": 2478646,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022512.108, "dur": 16449.587,
    "args": {
      "External id": 2478647,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022514.612, "dur": 16446.353,
    "args": {
      "External id": 2478648,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022520.053, "dur": 4.509,
    "args": {
      "External id": 2478649,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656022543.474, "dur": 16414.732,
    "args": {
      "External id": 2478650,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039209.261, "dur": 29.920,
    "args": {
      "External id": 2478651,"Sequence number": 24740429, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6354
    }
  },
  {
    "ph": "s", "id": 219, "pid": 1336757, "tid": 1336757, "ts": 1514656039209.261,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039224.620, "dur": 9.935,
    "args": {
      "External id": 2478652,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039229.594, "dur": 4.749,
    "args": {
      "External id": 2478653,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039300.409, "dur": 78.927,
    "args": {
      "External id": 2478654,"Record function id": 0, "Ev Idx": 6357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039380.673, "dur": 1105.401,
    "args": {
      "External id": 2478655,"Record function id": 0, "Ev Idx": 6358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039418.848, "dur": 1049.047,
    "args": {
      "External id": 2478656,"Sequence number": 24740430, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6359
    }
  },
  {
    "ph": "s", "id": 218, "pid": 1336757, "tid": 1336757, "ts": 1514656039418.848,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039482.250, "dur": 40.640,
    "args": {
      "External id": 2478657,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039554.818, "dur": 102.535,
    "args": {
      "External id": 2478658,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039668.144, "dur": 37.248,
    "args": {
      "External id": 2478659,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039715.279, "dur": 30.911,
    "args": {
      "External id": 2478660,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039771.940, "dur": 25.961,
    "args": {
      "External id": 2478661,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039814.720, "dur": 14.531,
    "args": {
      "External id": 2478662,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039852.427, "dur": 173.923,
    "args": {
      "External id": 2478663,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039902.561, "dur": 10.941,
    "args": {
      "External id": 2478664,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039907.212, "dur": 5.347,
    "args": {
      "External id": 2478665,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039915.918, "dur": 6.640,
    "args": {
      "External id": 2478666,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039925.172, "dur": 1.189,
    "args": {
      "External id": 2478667,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656039928.434, "dur": 4.017,
    "args": {
      "External id": 2478668,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656040041.585, "dur": 50.883,
    "args": {
      "External id": 2478669,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656040122.720, "dur": 31.023,
    "args": {
      "External id": 2478670,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656040162.716, "dur": 40.393,
    "args": {
      "External id": 2478671,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656040211.073, "dur": 34.542,
    "args": {
      "External id": 2478672,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656040266.222, "dur": 28.111,
    "args": {
      "External id": 2478673,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656040300.991, "dur": 33.536,
    "args": {
      "External id": 2478674,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656040354.366, "dur": 20.864,
    "args": {
      "External id": 2478675,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6378
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.4)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656040595.842, "dur": 101.275,
    "args": {
      "External id": 2478676,"Record function id": 0, "Ev Idx": 6379
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656040796.117, "dur": 61.058,
    "args": {
      "External id": 2478677,"Record function id": 0, "Ev Idx": 6380
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.5)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656040869.408, "dur": 18379.525,
    "args": {
      "External id": 2478678,"Record function id": 0, "Ev Idx": 6381
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.5)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656040882.845, "dur": 1049.460,
    "args": {
      "External id": 2478679,"Record function id": 0, "Ev Idx": 6382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656040972.670, "dur": 55.330,
    "args": {
      "External id": 2478680,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041044.322, "dur": 40.551,
    "args": {
      "External id": 2478681,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041050.737, "dur": 2.454,
    "args": {
      "External id": 2478682,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041056.799, "dur": 0.414,
    "args": {
      "External id": 2478683,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041059.798, "dur": 0.488,
    "args": {
      "External id": 2478684,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041061.006, "dur": 0.700,
    "args": {
      "External id": 2478685,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041064.084, "dur": 0.504,
    "args": {
      "External id": 2478686,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041067.557, "dur": 0.706,
    "args": {
      "External id": 2478687,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041068.853, "dur": 3.999,
    "args": {
      "External id": 2478688,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041073.553, "dur": 0.455,
    "args": {
      "External id": 2478689,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041076.397, "dur": 0.827,
    "args": {
      "External id": 2478690,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041096.190, "dur": 49.156,
    "args": {
      "External id": 2478691,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041179.892, "dur": 154.935,
    "args": {
      "External id": 2478692,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041194.289, "dur": 5.990,
    "args": {
      "External id": 2478693,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041206.638, "dur": 12.233,
    "args": {
      "External id": 2478694,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041211.705, "dur": 6.612,
    "args": {
      "External id": 2478695,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041215.571, "dur": 0.944,
    "args": {
      "External id": 2478696,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041227.486, "dur": 41.255,
    "args": {
      "External id": 2478697,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041231.571, "dur": 2.603,
    "args": {
      "External id": 2478698,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041235.667, "dur": 0.447,
    "args": {
      "External id": 2478699,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041237.343, "dur": 0.440,
    "args": {
      "External id": 2478700,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041242.404, "dur": 2.329,
    "args": {
      "External id": 2478701,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041246.103, "dur": 0.735,
    "args": {
      "External id": 2478702,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041248.300, "dur": 0.595,
    "args": {
      "External id": 2478703,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041254.394, "dur": 1.084,
    "args": {
      "External id": 2478704,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041256.836, "dur": 0.366,
    "args": {
      "External id": 2478705,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041259.416, "dur": 2.114,
    "args": {
      "External id": 2478706,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041288.234, "dur": 35.068,
    "args": {
      "External id": 2478707,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041408.992, "dur": 416.138,
    "args": {
      "External id": 2478708,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041445.659, "dur": 373.795,
    "args": {
      "External id": 2478709,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6412, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041460.832, "dur": 351.948,
    "args": {
      "External id": 2478710,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041853.832, "dur": 2.905,
    "args": {
      "External id": 2478711,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6414, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.5)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656041952.624, "dur": 17093.494,
    "args": {
      "External id": 2478712,"Record function id": 0, "Ev Idx": 6415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042099.855, "dur": 7.149,
    "args": {
      "External id": 2478713,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042110.546, "dur": 1.414,
    "args": {
      "External id": 2478714,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042115.116, "dur": 2.645,
    "args": {
      "External id": 2478715,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042119.618, "dur": 1.004,
    "args": {
      "External id": 2478716,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042122.471, "dur": 1.099,
    "args": {
      "External id": 2478717,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042125.041, "dur": 0.843,
    "args": {
      "External id": 2478718,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042130.034, "dur": 1.212,
    "args": {
      "External id": 2478719,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042133.012, "dur": 2.719,
    "args": {
      "External id": 2478720,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042137.222, "dur": 0.801,
    "args": {
      "External id": 2478721,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042139.419, "dur": 0.971,
    "args": {
      "External id": 2478722,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042160.831, "dur": 16802.297,
    "args": {
      "External id": 2478723,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042184.304, "dur": 16770.356,
    "args": {
      "External id": 2478724,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042204.749, "dur": 13.611,
    "args": {
      "External id": 2478725,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042221.619, "dur": 16697.155,
    "args": {
      "External id": 2478726,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042224.095, "dur": 16693.822,
    "args": {
      "External id": 2478727,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042230.252, "dur": 5.055,
    "args": {
      "External id": 2478728,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656042236.925, "dur": 16677.451,
    "args": {
      "External id": 2478729,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059194.012, "dur": 31.073,
    "args": {
      "External id": 2478730,"Sequence number": 24740431, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6433
    }
  },
  {
    "ph": "s", "id": 217, "pid": 1336757, "tid": 1336757, "ts": 1514656059194.012,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059210.892, "dur": 9.447,
    "args": {
      "External id": 2478731,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059214.774, "dur": 5.387,
    "args": {
      "External id": 2478732,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059286.934, "dur": 79.791,
    "args": {
      "External id": 2478733,"Record function id": 0, "Ev Idx": 6436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059369.032, "dur": 1079.087,
    "args": {
      "External id": 2478734,"Record function id": 0, "Ev Idx": 6437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059412.332, "dur": 1022.463,
    "args": {
      "External id": 2478735,"Sequence number": 24740432, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6438
    }
  },
  {
    "ph": "s", "id": 216, "pid": 1336757, "tid": 1336757, "ts": 1514656059412.332,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059477.914, "dur": 42.986,
    "args": {
      "External id": 2478736,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059551.875, "dur": 107.449,
    "args": {
      "External id": 2478737,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059671.905, "dur": 36.889,
    "args": {
      "External id": 2478738,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059717.053, "dur": 30.580,
    "args": {
      "External id": 2478739,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059772.991, "dur": 26.903,
    "args": {
      "External id": 2478740,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059818.705, "dur": 15.094,
    "args": {
      "External id": 2478741,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059858.730, "dur": 167.514,
    "args": {
      "External id": 2478742,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059907.052, "dur": 11.510,
    "args": {
      "External id": 2478743,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059912.070, "dur": 5.569,
    "args": {
      "External id": 2478744,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059921.486, "dur": 5.834,
    "args": {
      "External id": 2478745,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059928.662, "dur": 0.860,
    "args": {
      "External id": 2478746,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656059932.039, "dur": 5.508,
    "args": {
      "External id": 2478747,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060039.599, "dur": 50.642,
    "args": {
      "External id": 2478748,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060123.337, "dur": 30.176,
    "args": {
      "External id": 2478749,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060162.821, "dur": 40.484,
    "args": {
      "External id": 2478750,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060212.380, "dur": 34.439,
    "args": {
      "External id": 2478751,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060267.027, "dur": 25.221,
    "args": {
      "External id": 2478752,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060297.609, "dur": 33.242,
    "args": {
      "External id": 2478753,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060351.596, "dur": 16.704,
    "args": {
      "External id": 2478754,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6457
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.5)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060511.524, "dur": 96.873,
    "args": {
      "External id": 2478755,"Record function id": 0, "Ev Idx": 6458
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060685.534, "dur": 47.242,
    "args": {
      "External id": 2478756,"Record function id": 0, "Ev Idx": 6459
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.6)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060743.029, "dur": 18008.807,
    "args": {
      "External id": 2478757,"Record function id": 0, "Ev Idx": 6460
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.6)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060751.610, "dur": 937.263,
    "args": {
      "External id": 2478758,"Record function id": 0, "Ev Idx": 6461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060834.109, "dur": 8.797,
    "args": {
      "External id": 2478759,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060855.648, "dur": 39.493,
    "args": {
      "External id": 2478760,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060860.967, "dur": 2.462,
    "args": {
      "External id": 2478761,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060867.761, "dur": 0.520,
    "args": {
      "External id": 2478762,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060869.754, "dur": 0.626,
    "args": {
      "External id": 2478763,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060872.052, "dur": 0.596,
    "args": {
      "External id": 2478764,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060875.473, "dur": 0.524,
    "args": {
      "External id": 2478765,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060877.528, "dur": 0.541,
    "args": {
      "External id": 2478766,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060879.650, "dur": 3.565,
    "args": {
      "External id": 2478767,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060884.775, "dur": 0.644,
    "args": {
      "External id": 2478768,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060886.641, "dur": 0.410,
    "args": {
      "External id": 2478769,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060905.611, "dur": 38.756,
    "args": {
      "External id": 2478770,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656060978.324, "dur": 163.105,
    "args": {
      "External id": 2478771,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061030.734, "dur": 5.800,
    "args": {
      "External id": 2478772,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061042.382, "dur": 11.296,
    "args": {
      "External id": 2478773,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061046.876, "dur": 6.379,
    "args": {
      "External id": 2478774,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061050.803, "dur": 0.717,
    "args": {
      "External id": 2478775,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061061.464, "dur": 32.890,
    "args": {
      "External id": 2478776,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061064.058, "dur": 2.504,
    "args": {
      "External id": 2478777,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061068.303, "dur": 0.393,
    "args": {
      "External id": 2478778,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061070.119, "dur": 0.479,
    "args": {
      "External id": 2478779,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061073.588, "dur": 2.192,
    "args": {
      "External id": 2478780,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061077.436, "dur": 0.487,
    "args": {
      "External id": 2478781,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061079.300, "dur": 0.523,
    "args": {
      "External id": 2478782,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061082.654, "dur": 0.546,
    "args": {
      "External id": 2478783,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061084.468, "dur": 0.458,
    "args": {
      "External id": 2478784,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061086.373, "dur": 2.148,
    "args": {
      "External id": 2478785,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061106.701, "dur": 25.768,
    "args": {
      "External id": 2478786,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061198.453, "dur": 390.258,
    "args": {
      "External id": 2478787,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061229.723, "dur": 353.496,
    "args": {
      "External id": 2478788,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6491, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061242.110, "dur": 335.015,
    "args": {
      "External id": 2478789,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061614.979, "dur": 2.546,
    "args": {
      "External id": 2478790,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6493, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.6)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061709.212, "dur": 16841.736,
    "args": {
      "External id": 2478791,"Record function id": 0, "Ev Idx": 6494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061807.149, "dur": 6.083,
    "args": {
      "External id": 2478792,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061817.102, "dur": 1.283,
    "args": {
      "External id": 2478793,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061820.382, "dur": 2.632,
    "args": {
      "External id": 2478794,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061825.115, "dur": 0.738,
    "args": {
      "External id": 2478795,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061827.458, "dur": 1.140,
    "args": {
      "External id": 2478796,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061830.209, "dur": 1.122,
    "args": {
      "External id": 2478797,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061835.036, "dur": 0.863,
    "args": {
      "External id": 2478798,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061837.431, "dur": 3.227,
    "args": {
      "External id": 2478799,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061842.233, "dur": 0.931,
    "args": {
      "External id": 2478800,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061844.560, "dur": 1.049,
    "args": {
      "External id": 2478801,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061865.628, "dur": 16624.731,
    "args": {
      "External id": 2478802,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061881.320, "dur": 16601.903,
    "args": {
      "External id": 2478803,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061904.444, "dur": 15.214,
    "args": {
      "External id": 2478804,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061922.634, "dur": 16529.106,
    "args": {
      "External id": 2478805,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061925.194, "dur": 16525.908,
    "args": {
      "External id": 2478806,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061931.236, "dur": 4.550,
    "args": {
      "External id": 2478807,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656061937.456, "dur": 16510.364,
    "args": {
      "External id": 2478808,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656078698.095, "dur": 29.814,
    "args": {
      "External id": 2478809,"Sequence number": 24740433, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6512
    }
  },
  {
    "ph": "s", "id": 215, "pid": 1336757, "tid": 1336757, "ts": 1514656078698.095,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656078714.476, "dur": 8.774,
    "args": {
      "External id": 2478810,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656078718.032, "dur": 4.953,
    "args": {
      "External id": 2478811,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656078791.178, "dur": 80.281,
    "args": {
      "External id": 2478812,"Record function id": 0, "Ev Idx": 6515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656078873.447, "dur": 1055.753,
    "args": {
      "External id": 2478813,"Record function id": 0, "Ev Idx": 6516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656078913.829, "dur": 1001.713,
    "args": {
      "External id": 2478814,"Sequence number": 24740434, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6517
    }
  },
  {
    "ph": "s", "id": 214, "pid": 1336757, "tid": 1336757, "ts": 1514656078913.829,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656078976.518, "dur": 78.563,
    "args": {
      "External id": 2478815,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079070.847, "dur": 104.539,
    "args": {
      "External id": 2478816,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079185.879, "dur": 36.453,
    "args": {
      "External id": 2478817,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079229.886, "dur": 31.147,
    "args": {
      "External id": 2478818,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079286.032, "dur": 26.174,
    "args": {
      "External id": 2478819,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079328.623, "dur": 14.212,
    "args": {
      "External id": 2478820,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079361.934, "dur": 128.635,
    "args": {
      "External id": 2478821,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079412.381, "dur": 12.389,
    "args": {
      "External id": 2478822,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079417.320, "dur": 6.607,
    "args": {
      "External id": 2478823,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079427.708, "dur": 5.670,
    "args": {
      "External id": 2478824,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079434.665, "dur": 0.924,
    "args": {
      "External id": 2478825,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079437.872, "dur": 3.733,
    "args": {
      "External id": 2478826,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079500.899, "dur": 61.305,
    "args": {
      "External id": 2478827,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079595.337, "dur": 30.365,
    "args": {
      "External id": 2478828,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079634.707, "dur": 43.661,
    "args": {
      "External id": 2478829,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079686.386, "dur": 33.202,
    "args": {
      "External id": 2478830,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079741.839, "dur": 26.997,
    "args": {
      "External id": 2478831,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079775.040, "dur": 33.039,
    "args": {
      "External id": 2478832,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656079827.303, "dur": 17.686,
    "args": {
      "External id": 2478833,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6536
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.6)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080029.856, "dur": 77.367,
    "args": {
      "External id": 2478834,"Record function id": 0, "Ev Idx": 6537
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080181.468, "dur": 43.711,
    "args": {
      "External id": 2478835,"Record function id": 0, "Ev Idx": 6538
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.7)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080234.303, "dur": 18312.922,
    "args": {
      "External id": 2478836,"Record function id": 0, "Ev Idx": 6539
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.7)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080242.366, "dur": 918.188,
    "args": {
      "External id": 2478837,"Record function id": 0, "Ev Idx": 6540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080326.052, "dur": 8.504,
    "args": {
      "External id": 2478838,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080347.745, "dur": 39.105,
    "args": {
      "External id": 2478839,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080353.512, "dur": 2.270,
    "args": {
      "External id": 2478840,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080359.944, "dur": 0.686,
    "args": {
      "External id": 2478841,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080362.413, "dur": 0.408,
    "args": {
      "External id": 2478842,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080364.347, "dur": 0.669,
    "args": {
      "External id": 2478843,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080367.869, "dur": 0.564,
    "args": {
      "External id": 2478844,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080369.724, "dur": 0.670,
    "args": {
      "External id": 2478845,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080371.688, "dur": 3.595,
    "args": {
      "External id": 2478846,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080376.523, "dur": 0.578,
    "args": {
      "External id": 2478847,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080378.605, "dur": 0.484,
    "args": {
      "External id": 2478848,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080398.194, "dur": 38.578,
    "args": {
      "External id": 2478849,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080470.098, "dur": 137.068,
    "args": {
      "External id": 2478850,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080482.613, "dur": 3.933,
    "args": {
      "External id": 2478851,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080491.397, "dur": 11.016,
    "args": {
      "External id": 2478852,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080495.891, "dur": 6.104,
    "args": {
      "External id": 2478853,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080500.023, "dur": 0.692,
    "args": {
      "External id": 2478854,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080509.454, "dur": 48.910,
    "args": {
      "External id": 2478855,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080511.548, "dur": 2.290,
    "args": {
      "External id": 2478856,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080515.896, "dur": 0.566,
    "args": {
      "External id": 2478857,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080518.166, "dur": 0.250,
    "args": {
      "External id": 2478858,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080521.659, "dur": 1.985,
    "args": {
      "External id": 2478859,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080524.800, "dur": 0.579,
    "args": {
      "External id": 2478860,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080541.834, "dur": 0.637,
    "args": {
      "External id": 2478861,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080546.169, "dur": 0.495,
    "args": {
      "External id": 2478862,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080548.040, "dur": 0.595,
    "args": {
      "External id": 2478863,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080550.297, "dur": 2.437,
    "args": {
      "External id": 2478864,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080573.144, "dur": 25.231,
    "args": {
      "External id": 2478865,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080661.839, "dur": 395.258,
    "args": {
      "External id": 2478866,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080692.001, "dur": 359.487,
    "args": {
      "External id": 2478867,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6570, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656080703.851, "dur": 341.572,
    "args": {
      "External id": 2478868,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081083.728, "dur": 2.640,
    "args": {
      "External id": 2478869,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6572, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.7)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081181.491, "dur": 17152.351,
    "args": {
      "External id": 2478870,"Record function id": 0, "Ev Idx": 6573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081291.285, "dur": 6.444,
    "args": {
      "External id": 2478871,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081301.491, "dur": 1.223,
    "args": {
      "External id": 2478872,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081304.809, "dur": 3.170,
    "args": {
      "External id": 2478873,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081311.190, "dur": 1.149,
    "args": {
      "External id": 2478874,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081314.153, "dur": 0.862,
    "args": {
      "External id": 2478875,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081316.404, "dur": 0.905,
    "args": {
      "External id": 2478876,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081318.794, "dur": 0.914,
    "args": {
      "External id": 2478877,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081322.902, "dur": 3.121,
    "args": {
      "External id": 2478878,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081327.367, "dur": 0.966,
    "args": {
      "External id": 2478879,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081329.921, "dur": 0.770,
    "args": {
      "External id": 2478880,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081349.724, "dur": 16932.134,
    "args": {
      "External id": 2478881,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081366.222, "dur": 16908.096,
    "args": {
      "External id": 2478882,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081385.643, "dur": 14.971,
    "args": {
      "External id": 2478883,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081405.131, "dur": 16835.165,
    "args": {
      "External id": 2478884,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081407.598, "dur": 16831.955,
    "args": {
      "External id": 2478885,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081413.228, "dur": 4.522,
    "args": {
      "External id": 2478886,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656081419.297, "dur": 16816.836,
    "args": {
      "External id": 2478887,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656098476.685, "dur": 31.435,
    "args": {
      "External id": 2478888,"Sequence number": 24740435, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6591
    }
  },
  {
    "ph": "s", "id": 213, "pid": 1336757, "tid": 1336757, "ts": 1514656098476.685,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656098494.853, "dur": 8.316,
    "args": {
      "External id": 2478889,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656098498.336, "dur": 4.648,
    "args": {
      "External id": 2478890,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656098586.762, "dur": 77.043,
    "args": {
      "External id": 2478891,"Record function id": 0, "Ev Idx": 6594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656098666.019, "dur": 1076.195,
    "args": {
      "External id": 2478892,"Record function id": 0, "Ev Idx": 6595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656098707.930, "dur": 1020.903,
    "args": {
      "External id": 2478893,"Sequence number": 24740436, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6596
    }
  },
  {
    "ph": "s", "id": 212, "pid": 1336757, "tid": 1336757, "ts": 1514656098707.930,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656098775.071, "dur": 42.027,
    "args": {
      "External id": 2478894,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656098830.065, "dur": 104.209,
    "args": {
      "External id": 2478895,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656098943.950, "dur": 73.521,
    "args": {
      "External id": 2478896,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099030.698, "dur": 34.955,
    "args": {
      "External id": 2478897,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099090.638, "dur": 26.661,
    "args": {
      "External id": 2478898,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099134.129, "dur": 15.263,
    "args": {
      "External id": 2478899,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099168.994, "dur": 136.337,
    "args": {
      "External id": 2478900,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099217.965, "dur": 11.886,
    "args": {
      "External id": 2478901,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099223.433, "dur": 5.400,
    "args": {
      "External id": 2478902,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099232.896, "dur": 6.864,
    "args": {
      "External id": 2478903,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099246.381, "dur": 1.009,
    "args": {
      "External id": 2478904,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099250.350, "dur": 4.592,
    "args": {
      "External id": 2478905,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099316.184, "dur": 45.215,
    "args": {
      "External id": 2478906,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099393.010, "dur": 27.381,
    "args": {
      "External id": 2478907,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099429.796, "dur": 39.207,
    "args": {
      "External id": 2478908,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099476.706, "dur": 33.685,
    "args": {
      "External id": 2478909,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099548.301, "dur": 29.774,
    "args": {
      "External id": 2478910,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099585.034, "dur": 36.925,
    "args": {
      "External id": 2478911,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099643.450, "dur": 17.789,
    "args": {
      "External id": 2478912,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6615
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.7)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099805.758, "dur": 74.037,
    "args": {
      "External id": 2478913,"Record function id": 0, "Ev Idx": 6616
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656099953.331, "dur": 85.299,
    "args": {
      "External id": 2478914,"Record function id": 0, "Ev Idx": 6617
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.8)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100050.225, "dur": 18028.188,
    "args": {
      "External id": 2478915,"Record function id": 0, "Ev Idx": 6618
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.8)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100059.021, "dur": 887.906,
    "args": {
      "External id": 2478916,"Record function id": 0, "Ev Idx": 6619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100142.266, "dur": 8.719,
    "args": {
      "External id": 2478917,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100164.812, "dur": 39.496,
    "args": {
      "External id": 2478918,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100170.224, "dur": 2.308,
    "args": {
      "External id": 2478919,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100176.512, "dur": 0.542,
    "args": {
      "External id": 2478920,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100178.540, "dur": 0.531,
    "args": {
      "External id": 2478921,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100180.729, "dur": 0.752,
    "args": {
      "External id": 2478922,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100184.503, "dur": 0.503,
    "args": {
      "External id": 2478923,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100186.218, "dur": 0.929,
    "args": {
      "External id": 2478924,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100188.741, "dur": 4.204,
    "args": {
      "External id": 2478925,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100194.368, "dur": 0.443,
    "args": {
      "External id": 2478926,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100196.218, "dur": 0.352,
    "args": {
      "External id": 2478927,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100215.760, "dur": 41.666,
    "args": {
      "External id": 2478928,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100291.346, "dur": 110.176,
    "args": {
      "External id": 2478929,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100303.626, "dur": 4.166,
    "args": {
      "External id": 2478930,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100312.928, "dur": 10.455,
    "args": {
      "External id": 2478931,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100317.547, "dur": 5.453,
    "args": {
      "External id": 2478932,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100321.124, "dur": 0.507,
    "args": {
      "External id": 2478933,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100329.882, "dur": 31.243,
    "args": {
      "External id": 2478934,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100331.726, "dur": 2.221,
    "args": {
      "External id": 2478935,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100335.358, "dur": 0.641,
    "args": {
      "External id": 2478936,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100337.409, "dur": 0.534,
    "args": {
      "External id": 2478937,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100341.090, "dur": 2.357,
    "args": {
      "External id": 2478938,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100344.847, "dur": 0.411,
    "args": {
      "External id": 2478939,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100346.567, "dur": 0.331,
    "args": {
      "External id": 2478940,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100349.440, "dur": 0.369,
    "args": {
      "External id": 2478941,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100351.344, "dur": 0.350,
    "args": {
      "External id": 2478942,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100353.123, "dur": 1.681,
    "args": {
      "External id": 2478943,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100371.945, "dur": 21.576,
    "args": {
      "External id": 2478944,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100454.943, "dur": 395.669,
    "args": {
      "External id": 2478945,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100483.890, "dur": 360.917,
    "args": {
      "External id": 2478946,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6649, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100495.628, "dur": 343.186,
    "args": {
      "External id": 2478947,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100876.016, "dur": 2.372,
    "args": {
      "External id": 2478948,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6651, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.8)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656100967.114, "dur": 16898.497,
    "args": {
      "External id": 2478949,"Record function id": 0, "Ev Idx": 6652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101105.602, "dur": 6.970,
    "args": {
      "External id": 2478950,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101116.785, "dur": 1.282,
    "args": {
      "External id": 2478951,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101120.434, "dur": 2.624,
    "args": {
      "External id": 2478952,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101124.848, "dur": 1.301,
    "args": {
      "External id": 2478953,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101127.568, "dur": 0.911,
    "args": {
      "External id": 2478954,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101129.768, "dur": 1.127,
    "args": {
      "External id": 2478955,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101134.212, "dur": 0.964,
    "args": {
      "External id": 2478956,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101137.130, "dur": 3.093,
    "args": {
      "External id": 2478957,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101141.732, "dur": 0.963,
    "args": {
      "External id": 2478958,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101144.228, "dur": 1.071,
    "args": {
      "External id": 2478959,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101175.379, "dur": 16645.123,
    "args": {
      "External id": 2478960,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101192.168, "dur": 16621.984,
    "args": {
      "External id": 2478961,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101212.096, "dur": 14.063,
    "args": {
      "External id": 2478962,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101229.314, "dur": 16554.747,
    "args": {
      "External id": 2478963,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101231.612, "dur": 16551.892,
    "args": {
      "External id": 2478964,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101237.530, "dur": 4.896,
    "args": {
      "External id": 2478965,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656101243.938, "dur": 16536.990,
    "args": {
      "External id": 2478966,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118024.967, "dur": 28.693,
    "args": {
      "External id": 2478967,"Sequence number": 24740437, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6670
    }
  },
  {
    "ph": "s", "id": 211, "pid": 1336757, "tid": 1336757, "ts": 1514656118024.967,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118041.226, "dur": 7.665,
    "args": {
      "External id": 2478968,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118044.264, "dur": 4.318,
    "args": {
      "External id": 2478969,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118115.748, "dur": 79.183,
    "args": {
      "External id": 2478970,"Record function id": 0, "Ev Idx": 6673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118196.741, "dur": 1071.407,
    "args": {
      "External id": 2478971,"Record function id": 0, "Ev Idx": 6674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118235.909, "dur": 1017.973,
    "args": {
      "External id": 2478972,"Sequence number": 24740438, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6675
    }
  },
  {
    "ph": "s", "id": 210, "pid": 1336757, "tid": 1336757, "ts": 1514656118235.909,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118300.169, "dur": 42.696,
    "args": {
      "External id": 2478973,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118356.087, "dur": 104.250,
    "args": {
      "External id": 2478974,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118470.214, "dur": 39.131,
    "args": {
      "External id": 2478975,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118519.872, "dur": 48.912,
    "args": {
      "External id": 2478976,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118597.666, "dur": 27.380,
    "args": {
      "External id": 2478977,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118641.499, "dur": 15.499,
    "args": {
      "External id": 2478978,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118677.328, "dur": 126.681,
    "args": {
      "External id": 2478979,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118726.875, "dur": 11.397,
    "args": {
      "External id": 2478980,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118731.659, "dur": 5.648,
    "args": {
      "External id": 2478981,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118741.001, "dur": 5.737,
    "args": {
      "External id": 2478982,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118748.053, "dur": 1.193,
    "args": {
      "External id": 2478983,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118751.649, "dur": 4.118,
    "args": {
      "External id": 2478984,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118815.004, "dur": 47.178,
    "args": {
      "External id": 2478985,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118891.083, "dur": 27.261,
    "args": {
      "External id": 2478986,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118927.825, "dur": 40.637,
    "args": {
      "External id": 2478987,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656118976.638, "dur": 74.148,
    "args": {
      "External id": 2478988,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119077.904, "dur": 27.252,
    "args": {
      "External id": 2478989,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119111.552, "dur": 35.592,
    "args": {
      "External id": 2478990,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119166.133, "dur": 18.146,
    "args": {
      "External id": 2478991,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6694
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.8)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119330.396, "dur": 79.734,
    "args": {
      "External id": 2478992,"Record function id": 0, "Ev Idx": 6695
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119482.479, "dur": 60.553,
    "args": {
      "External id": 2478993,"Record function id": 0, "Ev Idx": 6696
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.9)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119555.271, "dur": 17945.522,
    "args": {
      "External id": 2478994,"Record function id": 0, "Ev Idx": 6697
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.9)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119562.896, "dur": 953.555,
    "args": {
      "External id": 2478995,"Record function id": 0, "Ev Idx": 6698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119646.255, "dur": 9.050,
    "args": {
      "External id": 2478996,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119668.329, "dur": 35.603,
    "args": {
      "External id": 2478997,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119673.700, "dur": 2.165,
    "args": {
      "External id": 2478998,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119679.753, "dur": 0.353,
    "args": {
      "External id": 2478999,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119681.039, "dur": 0.465,
    "args": {
      "External id": 2479000,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119682.708, "dur": 0.436,
    "args": {
      "External id": 2479001,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119686.145, "dur": 0.554,
    "args": {
      "External id": 2479002,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119688.226, "dur": 0.401,
    "args": {
      "External id": 2479003,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119689.607, "dur": 4.019,
    "args": {
      "External id": 2479004,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119694.724, "dur": 0.371,
    "args": {
      "External id": 2479005,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119696.079, "dur": 0.318,
    "args": {
      "External id": 2479006,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119713.897, "dur": 39.841,
    "args": {
      "External id": 2479007,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119787.524, "dur": 117.682,
    "args": {
      "External id": 2479008,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119799.709, "dur": 3.475,
    "args": {
      "External id": 2479009,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119808.419, "dur": 10.285,
    "args": {
      "External id": 2479010,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119812.971, "dur": 5.313,
    "args": {
      "External id": 2479011,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119816.414, "dur": 0.504,
    "args": {
      "External id": 2479012,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119824.539, "dur": 32.140,
    "args": {
      "External id": 2479013,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119826.371, "dur": 2.406,
    "args": {
      "External id": 2479014,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119832.859, "dur": 0.749,
    "args": {
      "External id": 2479015,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119834.757, "dur": 0.383,
    "args": {
      "External id": 2479016,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119838.027, "dur": 2.095,
    "args": {
      "External id": 2479017,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119840.920, "dur": 0.351,
    "args": {
      "External id": 2479018,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119842.182, "dur": 1.824,
    "args": {
      "External id": 2479019,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119844.950, "dur": 0.588,
    "args": {
      "External id": 2479020,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119846.708, "dur": 0.299,
    "args": {
      "External id": 2479021,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119849.801, "dur": 0.372,
    "args": {
      "External id": 2479022,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119875.438, "dur": 21.494,
    "args": {
      "External id": 2479023,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656119958.968, "dur": 456.154,
    "args": {
      "External id": 2479024,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120028.206, "dur": 381.394,
    "args": {
      "External id": 2479025,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6728, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120042.069, "dur": 361.551,
    "args": {
      "External id": 2479026,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120438.603, "dur": 2.695,
    "args": {
      "External id": 2479027,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6730, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.9)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120554.464, "dur": 16760.332,
    "args": {
      "External id": 2479028,"Record function id": 0, "Ev Idx": 6731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120662.155, "dur": 6.911,
    "args": {
      "External id": 2479029,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120673.571, "dur": 1.568,
    "args": {
      "External id": 2479030,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120677.244, "dur": 2.821,
    "args": {
      "External id": 2479031,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120681.808, "dur": 1.297,
    "args": {
      "External id": 2479032,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120684.550, "dur": 1.133,
    "args": {
      "External id": 2479033,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120687.172, "dur": 0.800,
    "args": {
      "External id": 2479034,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120691.912, "dur": 0.938,
    "args": {
      "External id": 2479035,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120694.232, "dur": 2.729,
    "args": {
      "External id": 2479036,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120698.296, "dur": 0.868,
    "args": {
      "External id": 2479037,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120700.357, "dur": 0.897,
    "args": {
      "External id": 2479038,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120720.813, "dur": 16549.493,
    "args": {
      "External id": 2479039,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120736.472, "dur": 16527.045,
    "args": {
      "External id": 2479040,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120757.981, "dur": 14.129,
    "args": {
      "External id": 2479041,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120775.085, "dur": 16456.828,
    "args": {
      "External id": 2479042,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120777.554, "dur": 16453.795,
    "args": {
      "External id": 2479043,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120783.080, "dur": 5.987,
    "args": {
      "External id": 2479044,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656120790.884, "dur": 16437.816,
    "args": {
      "External id": 2479045,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656137450.078, "dur": 27.608,
    "args": {
      "External id": 2479046,"Sequence number": 24740439, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6749
    }
  },
  {
    "ph": "s", "id": 209, "pid": 1336757, "tid": 1336757, "ts": 1514656137450.078,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656137465.700, "dur": 7.277,
    "args": {
      "External id": 2479047,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656137468.355, "dur": 4.419,
    "args": {
      "External id": 2479048,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656137551.623, "dur": 80.237,
    "args": {
      "External id": 2479049,"Record function id": 0, "Ev Idx": 6752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656137634.136, "dur": 1066.619,
    "args": {
      "External id": 2479050,"Record function id": 0, "Ev Idx": 6753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656137676.188, "dur": 1011.402,
    "args": {
      "External id": 2479051,"Sequence number": 24740440, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6754
    }
  },
  {
    "ph": "s", "id": 208, "pid": 1336757, "tid": 1336757, "ts": 1514656137676.188,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656137742.203, "dur": 40.870,
    "args": {
      "External id": 2479052,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656137795.751, "dur": 101.136,
    "args": {
      "External id": 2479053,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656137906.617, "dur": 39.505,
    "args": {
      "External id": 2479054,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656137952.878, "dur": 66.275,
    "args": {
      "External id": 2479055,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138049.705, "dur": 32.938,
    "args": {
      "External id": 2479056,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138098.963, "dur": 15.119,
    "args": {
      "External id": 2479057,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138133.605, "dur": 127.910,
    "args": {
      "External id": 2479058,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138183.687, "dur": 11.106,
    "args": {
      "External id": 2479059,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138188.490, "dur": 5.515,
    "args": {
      "External id": 2479060,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138197.509, "dur": 5.438,
    "args": {
      "External id": 2479061,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138204.099, "dur": 0.908,
    "args": {
      "External id": 2479062,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138207.520, "dur": 6.059,
    "args": {
      "External id": 2479063,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138272.647, "dur": 47.171,
    "args": {
      "External id": 2479064,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138347.081, "dur": 26.950,
    "args": {
      "External id": 2479065,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138388.872, "dur": 40.262,
    "args": {
      "External id": 2479066,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138437.771, "dur": 34.479,
    "args": {
      "External id": 2479067,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138495.591, "dur": 28.223,
    "args": {
      "External id": 2479068,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138545.549, "dur": 37.551,
    "args": {
      "External id": 2479069,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138604.553, "dur": 17.305,
    "args": {
      "External id": 2479070,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6773
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.9)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138765.038, "dur": 79.329,
    "args": {
      "External id": 2479071,"Record function id": 0, "Ev Idx": 6774
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138916.856, "dur": 45.200,
    "args": {
      "External id": 2479072,"Record function id": 0, "Ev Idx": 6775
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.10)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138971.793, "dur": 18061.747,
    "args": {
      "External id": 2479073,"Record function id": 0, "Ev Idx": 6776
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.10)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656138978.895, "dur": 930.339,
    "args": {
      "External id": 2479074,"Record function id": 0, "Ev Idx": 6777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139101.853, "dur": 9.165,
    "args": {
      "External id": 2479075,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139123.914, "dur": 34.210,
    "args": {
      "External id": 2479076,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139128.852, "dur": 2.351,
    "args": {
      "External id": 2479077,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139134.803, "dur": 0.549,
    "args": {
      "External id": 2479078,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139136.291, "dur": 0.465,
    "args": {
      "External id": 2479079,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139137.560, "dur": 0.645,
    "args": {
      "External id": 2479080,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139140.916, "dur": 0.518,
    "args": {
      "External id": 2479081,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139142.481, "dur": 0.495,
    "args": {
      "External id": 2479082,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139143.986, "dur": 2.369,
    "args": {
      "External id": 2479083,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139147.350, "dur": 0.450,
    "args": {
      "External id": 2479084,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139148.818, "dur": 0.474,
    "args": {
      "External id": 2479085,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139169.036, "dur": 39.155,
    "args": {
      "External id": 2479086,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139242.874, "dur": 111.429,
    "args": {
      "External id": 2479087,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139255.542, "dur": 3.428,
    "args": {
      "External id": 2479088,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139264.024, "dur": 11.008,
    "args": {
      "External id": 2479089,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139268.593, "dur": 6.047,
    "args": {
      "External id": 2479090,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139272.599, "dur": 0.501,
    "args": {
      "External id": 2479091,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139281.768, "dur": 32.044,
    "args": {
      "External id": 2479092,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139284.232, "dur": 2.132,
    "args": {
      "External id": 2479093,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139287.859, "dur": 0.425,
    "args": {
      "External id": 2479094,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139289.908, "dur": 0.578,
    "args": {
      "External id": 2479095,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139293.565, "dur": 1.727,
    "args": {
      "External id": 2479096,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139296.859, "dur": 0.360,
    "args": {
      "External id": 2479097,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139298.628, "dur": 0.586,
    "args": {
      "External id": 2479098,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139302.198, "dur": 0.334,
    "args": {
      "External id": 2479099,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139303.904, "dur": 0.483,
    "args": {
      "External id": 2479100,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139305.956, "dur": 2.187,
    "args": {
      "External id": 2479101,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139324.802, "dur": 21.304,
    "args": {
      "External id": 2479102,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139406.054, "dur": 402.988,
    "args": {
      "External id": 2479103,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139436.718, "dur": 367.061,
    "args": {
      "External id": 2479104,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6807, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139473.132, "dur": 323.547,
    "args": {
      "External id": 2479105,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139835.165, "dur": 2.533,
    "args": {
      "External id": 2479106,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6809, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.10)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656139930.404, "dur": 16871.117,
    "args": {
      "External id": 2479107,"Record function id": 0, "Ev Idx": 6810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140069.013, "dur": 6.890,
    "args": {
      "External id": 2479108,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140079.967, "dur": 1.150,
    "args": {
      "External id": 2479109,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140083.093, "dur": 2.795,
    "args": {
      "External id": 2479110,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140087.781, "dur": 1.192,
    "args": {
      "External id": 2479111,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140091.159, "dur": 1.023,
    "args": {
      "External id": 2479112,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140093.696, "dur": 0.898,
    "args": {
      "External id": 2479113,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140097.961, "dur": 1.417,
    "args": {
      "External id": 2479114,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140101.221, "dur": 2.196,
    "args": {
      "External id": 2479115,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140105.196, "dur": 1.000,
    "args": {
      "External id": 2479116,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140107.606, "dur": 0.769,
    "args": {
      "External id": 2479117,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140129.952, "dur": 16627.782,
    "args": {
      "External id": 2479118,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140146.624, "dur": 16603.524,
    "args": {
      "External id": 2479119,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140165.798, "dur": 14.436,
    "args": {
      "External id": 2479120,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140183.395, "dur": 16533.425,
    "args": {
      "External id": 2479121,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140186.013, "dur": 16530.058,
    "args": {
      "External id": 2479122,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140191.735, "dur": 6.017,
    "args": {
      "External id": 2479123,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656140199.509, "dur": 16513.392,
    "args": {
      "External id": 2479124,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656156944.373, "dur": 35.961,
    "args": {
      "External id": 2479125,"Sequence number": 24740441, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6828
    }
  },
  {
    "ph": "s", "id": 207, "pid": 1336757, "tid": 1336757, "ts": 1514656156944.373,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656156967.197, "dur": 8.656,
    "args": {
      "External id": 2479126,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656156971.025, "dur": 4.594,
    "args": {
      "External id": 2479127,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157073.056, "dur": 78.740,
    "args": {
      "External id": 2479128,"Record function id": 0, "Ev Idx": 6831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157153.484, "dur": 1069.692,
    "args": {
      "External id": 2479129,"Record function id": 0, "Ev Idx": 6832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157196.195, "dur": 1013.531,
    "args": {
      "External id": 2479130,"Sequence number": 24740442, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6833
    }
  },
  {
    "ph": "s", "id": 206, "pid": 1336757, "tid": 1336757, "ts": 1514656157196.195,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157261.229, "dur": 42.576,
    "args": {
      "External id": 2479131,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157317.215, "dur": 105.232,
    "args": {
      "External id": 2479132,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157432.662, "dur": 38.497,
    "args": {
      "External id": 2479133,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157478.770, "dur": 30.744,
    "args": {
      "External id": 2479134,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157550.530, "dur": 28.271,
    "args": {
      "External id": 2479135,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157597.581, "dur": 15.597,
    "args": {
      "External id": 2479136,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157632.059, "dur": 132.379,
    "args": {
      "External id": 2479137,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157683.462, "dur": 11.322,
    "args": {
      "External id": 2479138,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157688.364, "dur": 5.678,
    "args": {
      "External id": 2479139,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157697.707, "dur": 6.011,
    "args": {
      "External id": 2479140,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157705.369, "dur": 1.014,
    "args": {
      "External id": 2479141,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157708.793, "dur": 5.551,
    "args": {
      "External id": 2479142,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157774.352, "dur": 47.111,
    "args": {
      "External id": 2479143,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157850.551, "dur": 26.803,
    "args": {
      "External id": 2479144,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157888.114, "dur": 40.766,
    "args": {
      "External id": 2479145,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656157936.673, "dur": 35.084,
    "args": {
      "External id": 2479146,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158029.250, "dur": 28.777,
    "args": {
      "External id": 2479147,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158065.466, "dur": 38.255,
    "args": {
      "External id": 2479148,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158124.008, "dur": 17.945,
    "args": {
      "External id": 2479149,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6852
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.10)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158287.180, "dur": 77.210,
    "args": {
      "External id": 2479150,"Record function id": 0, "Ev Idx": 6853
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158435.662, "dur": 45.650,
    "args": {
      "External id": 2479151,"Record function id": 0, "Ev Idx": 6854
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.11)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158490.798, "dur": 18069.947,
    "args": {
      "External id": 2479152,"Record function id": 0, "Ev Idx": 6855
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.11)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158499.550, "dur": 915.416,
    "args": {
      "External id": 2479153,"Record function id": 0, "Ev Idx": 6856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158599.918, "dur": 9.238,
    "args": {
      "External id": 2479154,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158622.824, "dur": 40.631,
    "args": {
      "External id": 2479155,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158628.423, "dur": 2.765,
    "args": {
      "External id": 2479156,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158635.743, "dur": 0.451,
    "args": {
      "External id": 2479157,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158638.387, "dur": 0.696,
    "args": {
      "External id": 2479158,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158640.840, "dur": 0.614,
    "args": {
      "External id": 2479159,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158644.617, "dur": 0.614,
    "args": {
      "External id": 2479160,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158646.600, "dur": 0.457,
    "args": {
      "External id": 2479161,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158648.780, "dur": 3.851,
    "args": {
      "External id": 2479162,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158654.328, "dur": 0.550,
    "args": {
      "External id": 2479163,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158656.234, "dur": 0.446,
    "args": {
      "External id": 2479164,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158677.069, "dur": 38.902,
    "args": {
      "External id": 2479165,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158750.677, "dur": 113.388,
    "args": {
      "External id": 2479166,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158763.326, "dur": 5.197,
    "args": {
      "External id": 2479167,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158773.496, "dur": 10.883,
    "args": {
      "External id": 2479168,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158778.216, "dur": 5.754,
    "args": {
      "External id": 2479169,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158782.013, "dur": 0.562,
    "args": {
      "External id": 2479170,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158791.429, "dur": 29.367,
    "args": {
      "External id": 2479171,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158793.502, "dur": 0.524,
    "args": {
      "External id": 2479172,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158795.804, "dur": 2.146,
    "args": {
      "External id": 2479173,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158799.593, "dur": 0.520,
    "args": {
      "External id": 2479174,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158801.815, "dur": 1.631,
    "args": {
      "External id": 2479175,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158806.952, "dur": 0.273,
    "args": {
      "External id": 2479176,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158808.542, "dur": 0.635,
    "args": {
      "External id": 2479177,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158810.390, "dur": 0.598,
    "args": {
      "External id": 2479178,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158814.056, "dur": 0.645,
    "args": {
      "External id": 2479179,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158816.093, "dur": 0.494,
    "args": {
      "External id": 2479180,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158832.751, "dur": 23.767,
    "args": {
      "External id": 2479181,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158916.038, "dur": 401.337,
    "args": {
      "External id": 2479182,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158945.103, "dur": 367.402,
    "args": {
      "External id": 2479183,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6886, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656158957.072, "dur": 349.431,
    "args": {
      "External id": 2479184,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159341.468, "dur": 2.575,
    "args": {
      "External id": 2479185,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6888, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.11)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159435.941, "dur": 16915.230,
    "args": {
      "External id": 2479186,"Record function id": 0, "Ev Idx": 6889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159559.345, "dur": 6.743,
    "args": {
      "External id": 2479187,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159569.730, "dur": 1.300,
    "args": {
      "External id": 2479188,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159572.701, "dur": 2.183,
    "args": {
      "External id": 2479189,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159576.821, "dur": 0.927,
    "args": {
      "External id": 2479190,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159579.016, "dur": 1.127,
    "args": {
      "External id": 2479191,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159581.848, "dur": 0.900,
    "args": {
      "External id": 2479192,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159586.570, "dur": 1.082,
    "args": {
      "External id": 2479193,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159589.190, "dur": 2.053,
    "args": {
      "External id": 2479194,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159592.598, "dur": 0.658,
    "args": {
      "External id": 2479195,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159595.051, "dur": 0.957,
    "args": {
      "External id": 2479196,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159616.848, "dur": 16696.321,
    "args": {
      "External id": 2479197,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159633.501, "dur": 16673.136,
    "args": {
      "External id": 2479198,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159656.421, "dur": 13.814,
    "args": {
      "External id": 2479199,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159673.011, "dur": 16603.020,
    "args": {
      "External id": 2479200,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159675.572, "dur": 16599.934,
    "args": {
      "External id": 2479201,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159681.629, "dur": 6.056,
    "args": {
      "External id": 2479202,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656159689.251, "dur": 16583.290,
    "args": {
      "External id": 2479203,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656176485.490, "dur": 34.608,
    "args": {
      "External id": 2479204,"Sequence number": 24740443, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6907
    }
  },
  {
    "ph": "s", "id": 205, "pid": 1336757, "tid": 1336757, "ts": 1514656176485.490,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656176507.804, "dur": 7.676,
    "args": {
      "External id": 2479205,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656176511.297, "dur": 3.995,
    "args": {
      "External id": 2479206,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656176599.717, "dur": 80.230,
    "args": {
      "External id": 2479207,"Record function id": 0, "Ev Idx": 6910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656176681.403, "dur": 1051.990,
    "args": {
      "External id": 2479208,"Record function id": 0, "Ev Idx": 6911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656176722.659, "dur": 997.495,
    "args": {
      "External id": 2479209,"Sequence number": 24740444, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6912
    }
  },
  {
    "ph": "s", "id": 204, "pid": 1336757, "tid": 1336757, "ts": 1514656176722.659,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656176787.338, "dur": 42.292,
    "args": {
      "External id": 2479210,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656176842.277, "dur": 101.544,
    "args": {
      "External id": 2479211,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656176953.207, "dur": 71.176,
    "args": {
      "External id": 2479212,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177034.968, "dur": 35.203,
    "args": {
      "External id": 2479213,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177097.893, "dur": 26.711,
    "args": {
      "External id": 2479214,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177141.234, "dur": 16.070,
    "args": {
      "External id": 2479215,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177176.575, "dur": 129.177,
    "args": {
      "External id": 2479216,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177226.904, "dur": 11.310,
    "args": {
      "External id": 2479217,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177231.825, "dur": 5.465,
    "args": {
      "External id": 2479218,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177241.265, "dur": 6.771,
    "args": {
      "External id": 2479219,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177249.617, "dur": 0.867,
    "args": {
      "External id": 2479220,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177252.997, "dur": 4.353,
    "args": {
      "External id": 2479221,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177316.506, "dur": 44.208,
    "args": {
      "External id": 2479222,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177388.954, "dur": 28.521,
    "args": {
      "External id": 2479223,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177425.843, "dur": 39.682,
    "args": {
      "External id": 2479224,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177473.951, "dur": 34.564,
    "args": {
      "External id": 2479225,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 6928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177543.565, "dur": 28.803,
    "args": {
      "External id": 2479226,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 6929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177578.885, "dur": 37.735,
    "args": {
      "External id": 2479227,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 6930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177638.043, "dur": 16.971,
    "args": {
      "External id": 2479228,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 6931
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.11)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177797.959, "dur": 78.285,
    "args": {
      "External id": 2479229,"Record function id": 0, "Ev Idx": 6932
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656177950.087, "dur": 85.170,
    "args": {
      "External id": 2479230,"Record function id": 0, "Ev Idx": 6933
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.12)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178047.050, "dur": 18056.914,
    "args": {
      "External id": 2479231,"Record function id": 0, "Ev Idx": 6934
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.12)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178057.299, "dur": 859.447,
    "args": {
      "External id": 2479232,"Record function id": 0, "Ev Idx": 6935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178139.046, "dur": 9.036,
    "args": {
      "External id": 2479233,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178164.084, "dur": 38.698,
    "args": {
      "External id": 2479234,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178170.019, "dur": 2.448,
    "args": {
      "External id": 2479235,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178176.789, "dur": 0.556,
    "args": {
      "External id": 2479236,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178178.703, "dur": 0.490,
    "args": {
      "External id": 2479237,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178180.652, "dur": 2.262,
    "args": {
      "External id": 2479238,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178184.297, "dur": 0.440,
    "args": {
      "External id": 2479239,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178186.369, "dur": 0.471,
    "args": {
      "External id": 2479240,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178190.277, "dur": 1.771,
    "args": {
      "External id": 2479241,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178193.423, "dur": 0.576,
    "args": {
      "External id": 2479242,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178195.356, "dur": 0.529,
    "args": {
      "External id": 2479243,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178213.702, "dur": 43.223,
    "args": {
      "External id": 2479244,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178290.088, "dur": 110.958,
    "args": {
      "External id": 2479245,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 6948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178302.177, "dur": 3.372,
    "args": {
      "External id": 2479246,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178310.378, "dur": 12.003,
    "args": {
      "External id": 2479247,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178315.132, "dur": 6.835,
    "args": {
      "External id": 2479248,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 6951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178318.537, "dur": 2.168,
    "args": {
      "External id": 2479249,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 6952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178329.435, "dur": 28.774,
    "args": {
      "External id": 2479250,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 6953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178331.783, "dur": 0.487,
    "args": {
      "External id": 2479251,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178333.788, "dur": 0.493,
    "args": {
      "External id": 2479252,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178335.687, "dur": 0.262,
    "args": {
      "External id": 2479253,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178338.992, "dur": 1.500,
    "args": {
      "External id": 2479254,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178342.016, "dur": 0.695,
    "args": {
      "External id": 2479255,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178344.009, "dur": 2.055,
    "args": {
      "External id": 2479256,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178347.679, "dur": 0.497,
    "args": {
      "External id": 2479257,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178349.838, "dur": 0.454,
    "args": {
      "External id": 2479258,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178353.978, "dur": 0.221,
    "args": {
      "External id": 2479259,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 6962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178368.450, "dur": 24.725,
    "args": {
      "External id": 2479260,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 6963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178452.500, "dur": 371.164,
    "args": {
      "External id": 2479261,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 6964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178480.867, "dur": 337.862,
    "args": {
      "External id": 2479262,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 6965, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178491.860, "dur": 321.349,
    "args": {
      "External id": 2479263,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 6966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178848.429, "dur": 2.242,
    "args": {
      "External id": 2479264,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 6967, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.12)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656178938.800, "dur": 16943.514,
    "args": {
      "External id": 2479265,"Record function id": 0, "Ev Idx": 6968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179072.115, "dur": 6.878,
    "args": {
      "External id": 2479266,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 6969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179082.944, "dur": 0.855,
    "args": {
      "External id": 2479267,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179085.400, "dur": 2.554,
    "args": {
      "External id": 2479268,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179089.763, "dur": 0.836,
    "args": {
      "External id": 2479269,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179091.972, "dur": 0.974,
    "args": {
      "External id": 2479270,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179094.406, "dur": 0.930,
    "args": {
      "External id": 2479271,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 6974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179098.746, "dur": 0.846,
    "args": {
      "External id": 2479272,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 6975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179101.207, "dur": 2.153,
    "args": {
      "External id": 2479273,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179104.930, "dur": 0.748,
    "args": {
      "External id": 2479274,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179107.167, "dur": 0.816,
    "args": {
      "External id": 2479275,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 6978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179128.363, "dur": 16717.123,
    "args": {
      "External id": 2479276,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179144.796, "dur": 16694.341,
    "args": {
      "External id": 2479277,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 6980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179168.062, "dur": 13.471,
    "args": {
      "External id": 2479278,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179184.639, "dur": 16623.258,
    "args": {
      "External id": 2479279,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 6982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179187.388, "dur": 16619.959,
    "args": {
      "External id": 2479280,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 6983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179193.190, "dur": 6.128,
    "args": {
      "External id": 2479281,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 6984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656179201.132, "dur": 16603.359,
    "args": {
      "External id": 2479282,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 6985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196041.650, "dur": 37.842,
    "args": {
      "External id": 2479283,"Sequence number": 24740445, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 6986
    }
  },
  {
    "ph": "s", "id": 203, "pid": 1336757, "tid": 1336757, "ts": 1514656196041.650,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196067.067, "dur": 7.595,
    "args": {
      "External id": 2479284,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 6987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196070.254, "dur": 4.065,
    "args": {
      "External id": 2479285,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 6988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196141.564, "dur": 81.345,
    "args": {
      "External id": 2479286,"Record function id": 0, "Ev Idx": 6989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196224.463, "dur": 1055.617,
    "args": {
      "External id": 2479287,"Record function id": 0, "Ev Idx": 6990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196264.239, "dur": 1002.286,
    "args": {
      "External id": 2479288,"Sequence number": 24740446, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 6991
    }
  },
  {
    "ph": "s", "id": 202, "pid": 1336757, "tid": 1336757, "ts": 1514656196264.239,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196328.628, "dur": 41.670,
    "args": {
      "External id": 2479289,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 6992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196382.941, "dur": 104.908,
    "args": {
      "External id": 2479290,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196497.251, "dur": 57.369,
    "args": {
      "External id": 2479291,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196565.081, "dur": 33.763,
    "args": {
      "External id": 2479292,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 6995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196623.784, "dur": 27.762,
    "args": {
      "External id": 2479293,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196670.136, "dur": 14.692,
    "args": {
      "External id": 2479294,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 6997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196703.287, "dur": 128.645,
    "args": {
      "External id": 2479295,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 6998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196754.299, "dur": 10.718,
    "args": {
      "External id": 2479296,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 6999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196759.068, "dur": 5.123,
    "args": {
      "External id": 2479297,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196768.234, "dur": 5.343,
    "args": {
      "External id": 2479298,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196774.803, "dur": 1.015,
    "args": {
      "External id": 2479299,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196778.763, "dur": 5.238,
    "args": {
      "External id": 2479300,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196842.541, "dur": 43.474,
    "args": {
      "External id": 2479301,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196911.925, "dur": 26.168,
    "args": {
      "External id": 2479302,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656196947.248, "dur": 74.553,
    "args": {
      "External id": 2479303,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197033.581, "dur": 37.745,
    "args": {
      "External id": 2479304,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197098.920, "dur": 28.223,
    "args": {
      "External id": 2479305,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197135.194, "dur": 33.710,
    "args": {
      "External id": 2479306,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197186.191, "dur": 18.078,
    "args": {
      "External id": 2479307,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7010
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.12)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197344.484, "dur": 75.994,
    "args": {
      "External id": 2479308,"Record function id": 0, "Ev Idx": 7011
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197491.036, "dur": 65.071,
    "args": {
      "External id": 2479309,"Record function id": 0, "Ev Idx": 7012
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.13)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197567.062, "dur": 18192.964,
    "args": {
      "External id": 2479310,"Record function id": 0, "Ev Idx": 7013
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.13)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197576.128, "dur": 878.375,
    "args": {
      "External id": 2479311,"Record function id": 0, "Ev Idx": 7014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197657.923, "dur": 9.491,
    "args": {
      "External id": 2479312,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197681.064, "dur": 36.182,
    "args": {
      "External id": 2479313,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197686.681, "dur": 2.419,
    "args": {
      "External id": 2479314,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197693.369, "dur": 0.348,
    "args": {
      "External id": 2479315,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197695.073, "dur": 0.464,
    "args": {
      "External id": 2479316,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197697.222, "dur": 0.226,
    "args": {
      "External id": 2479317,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197700.079, "dur": 0.455,
    "args": {
      "External id": 2479318,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197702.108, "dur": 0.220,
    "args": {
      "External id": 2479319,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197703.798, "dur": 3.133,
    "args": {
      "External id": 2479320,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197708.441, "dur": 0.198,
    "args": {
      "External id": 2479321,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197710.326, "dur": 0.220,
    "args": {
      "External id": 2479322,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197728.428, "dur": 42.696,
    "args": {
      "External id": 2479323,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197804.448, "dur": 112.474,
    "args": {
      "External id": 2479324,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197816.127, "dur": 3.663,
    "args": {
      "External id": 2479325,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197824.752, "dur": 10.455,
    "args": {
      "External id": 2479326,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197829.720, "dur": 5.096,
    "args": {
      "External id": 2479327,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197833.180, "dur": 0.445,
    "args": {
      "External id": 2479328,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197842.020, "dur": 29.391,
    "args": {
      "External id": 2479329,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197844.525, "dur": 1.915,
    "args": {
      "External id": 2479330,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197847.845, "dur": 0.421,
    "args": {
      "External id": 2479331,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197849.566, "dur": 0.513,
    "args": {
      "External id": 2479332,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197853.444, "dur": 1.375,
    "args": {
      "External id": 2479333,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197856.228, "dur": 0.468,
    "args": {
      "External id": 2479334,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197858.192, "dur": 0.411,
    "args": {
      "External id": 2479335,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197861.466, "dur": 0.347,
    "args": {
      "External id": 2479336,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197863.190, "dur": 0.668,
    "args": {
      "External id": 2479337,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197865.029, "dur": 1.969,
    "args": {
      "External id": 2479338,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197884.133, "dur": 25.316,
    "args": {
      "External id": 2479339,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656197969.421, "dur": 391.599,
    "args": {
      "External id": 2479340,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198040.096, "dur": 316.086,
    "args": {
      "External id": 2479341,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7044, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198051.847, "dur": 298.732,
    "args": {
      "External id": 2479342,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198385.513, "dur": 2.435,
    "args": {
      "External id": 2479343,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7046, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.13)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198474.791, "dur": 17090.404,
    "args": {
      "External id": 2479344,"Record function id": 0, "Ev Idx": 7047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198595.502, "dur": 7.234,
    "args": {
      "External id": 2479345,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198606.577, "dur": 1.081,
    "args": {
      "External id": 2479346,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198609.330, "dur": 2.453,
    "args": {
      "External id": 2479347,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198613.402, "dur": 0.973,
    "args": {
      "External id": 2479348,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198615.780, "dur": 0.888,
    "args": {
      "External id": 2479349,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198617.938, "dur": 1.014,
    "args": {
      "External id": 2479350,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198622.628, "dur": 0.835,
    "args": {
      "External id": 2479351,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198625.014, "dur": 1.933,
    "args": {
      "External id": 2479352,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198628.652, "dur": 0.921,
    "args": {
      "External id": 2479353,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198631.180, "dur": 0.705,
    "args": {
      "External id": 2479354,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198652.326, "dur": 16862.653,
    "args": {
      "External id": 2479355,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198668.515, "dur": 16839.689,
    "args": {
      "External id": 2479356,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198693.188, "dur": 13.342,
    "args": {
      "External id": 2479357,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198709.377, "dur": 16768.068,
    "args": {
      "External id": 2479358,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198712.072, "dur": 16764.832,
    "args": {
      "External id": 2479359,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198718.247, "dur": 5.696,
    "args": {
      "External id": 2479360,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656198725.556, "dur": 16748.624,
    "args": {
      "External id": 2479361,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656215701.561, "dur": 34.671,
    "args": {
      "External id": 2479362,"Sequence number": 24740447, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7065
    }
  },
  {
    "ph": "s", "id": 201, "pid": 1336757, "tid": 1336757, "ts": 1514656215701.561,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656215723.468, "dur": 8.167,
    "args": {
      "External id": 2479363,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656215726.848, "dur": 4.533,
    "args": {
      "External id": 2479364,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656215797.231, "dur": 76.637,
    "args": {
      "External id": 2479365,"Record function id": 0, "Ev Idx": 7068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656215875.514, "dur": 1035.831,
    "args": {
      "External id": 2479366,"Record function id": 0, "Ev Idx": 7069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656215914.614, "dur": 983.213,
    "args": {
      "External id": 2479367,"Sequence number": 24740448, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7070
    }
  },
  {
    "ph": "s", "id": 200, "pid": 1336757, "tid": 1336757, "ts": 1514656215914.614,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656215976.600, "dur": 74.077,
    "args": {
      "External id": 2479368,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216065.506, "dur": 101.781,
    "args": {
      "External id": 2479369,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216176.660, "dur": 37.971,
    "args": {
      "External id": 2479370,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216223.785, "dur": 29.884,
    "args": {
      "External id": 2479371,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216278.373, "dur": 24.874,
    "args": {
      "External id": 2479372,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216320.036, "dur": 15.238,
    "args": {
      "External id": 2479373,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216354.204, "dur": 128.285,
    "args": {
      "External id": 2479374,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216403.713, "dur": 11.352,
    "args": {
      "External id": 2479375,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216408.596, "dur": 5.706,
    "args": {
      "External id": 2479376,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216417.930, "dur": 5.751,
    "args": {
      "External id": 2479377,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216425.298, "dur": 1.227,
    "args": {
      "External id": 2479378,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216428.854, "dur": 4.087,
    "args": {
      "External id": 2479379,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216493.304, "dur": 58.864,
    "args": {
      "External id": 2479380,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216584.700, "dur": 26.775,
    "args": {
      "External id": 2479381,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216620.790, "dur": 43.100,
    "args": {
      "External id": 2479382,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216671.927, "dur": 33.822,
    "args": {
      "External id": 2479383,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216728.248, "dur": 25.578,
    "args": {
      "External id": 2479384,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216759.635, "dur": 32.812,
    "args": {
      "External id": 2479385,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216810.315, "dur": 16.573,
    "args": {
      "External id": 2479386,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7089
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.13)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656216973.272, "dur": 117.576,
    "args": {
      "External id": 2479387,"Record function id": 0, "Ev Idx": 7090
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217166.450, "dur": 46.687,
    "args": {
      "External id": 2479388,"Record function id": 0, "Ev Idx": 7091
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.14)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217223.408, "dur": 17976.900,
    "args": {
      "External id": 2479389,"Record function id": 0, "Ev Idx": 7092
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.14)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217231.541, "dur": 943.509,
    "args": {
      "External id": 2479390,"Record function id": 0, "Ev Idx": 7093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217313.722, "dur": 8.401,
    "args": {
      "External id": 2479391,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217334.176, "dur": 44.008,
    "args": {
      "External id": 2479392,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217343.356, "dur": 2.379,
    "args": {
      "External id": 2479393,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217350.553, "dur": 0.454,
    "args": {
      "External id": 2479394,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217352.362, "dur": 0.171,
    "args": {
      "External id": 2479395,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217353.975, "dur": 2.023,
    "args": {
      "External id": 2479396,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217357.886, "dur": 0.419,
    "args": {
      "External id": 2479397,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217361.555, "dur": 0.299,
    "args": {
      "External id": 2479398,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217364.539, "dur": 1.611,
    "args": {
      "External id": 2479399,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217367.562, "dur": 0.406,
    "args": {
      "External id": 2479400,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217369.215, "dur": 2.396,
    "args": {
      "External id": 2479401,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217388.775, "dur": 41.251,
    "args": {
      "External id": 2479402,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217462.963, "dur": 130.734,
    "args": {
      "External id": 2479403,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217473.039, "dur": 3.457,
    "args": {
      "External id": 2479404,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217481.799, "dur": 10.760,
    "args": {
      "External id": 2479405,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217486.589, "dur": 5.510,
    "args": {
      "External id": 2479406,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217490.359, "dur": 0.434,
    "args": {
      "External id": 2479407,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217499.320, "dur": 42.874,
    "args": {
      "External id": 2479408,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217501.535, "dur": 0.611,
    "args": {
      "External id": 2479409,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217503.759, "dur": 0.555,
    "args": {
      "External id": 2479410,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217505.603, "dur": 2.274,
    "args": {
      "External id": 2479411,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217509.413, "dur": 1.427,
    "args": {
      "External id": 2479412,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217512.066, "dur": 0.299,
    "args": {
      "External id": 2479413,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217515.633, "dur": 0.177,
    "args": {
      "External id": 2479414,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217517.333, "dur": 0.343,
    "args": {
      "External id": 2479415,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217518.949, "dur": 0.348,
    "args": {
      "External id": 2479416,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217522.083, "dur": 0.440,
    "args": {
      "External id": 2479417,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217556.097, "dur": 29.061,
    "args": {
      "External id": 2479418,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217647.945, "dur": 426.948,
    "args": {
      "External id": 2479419,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217679.853, "dur": 389.254,
    "args": {
      "External id": 2479420,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7123, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656217690.685, "dur": 372.222,
    "args": {
      "External id": 2479421,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218100.351, "dur": 2.578,
    "args": {
      "External id": 2479422,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7125, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.14)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218196.753, "dur": 16779.460,
    "args": {
      "External id": 2479423,"Record function id": 0, "Ev Idx": 7126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218297.374, "dur": 6.589,
    "args": {
      "External id": 2479424,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218307.843, "dur": 1.136,
    "args": {
      "External id": 2479425,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218310.716, "dur": 2.759,
    "args": {
      "External id": 2479426,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218315.340, "dur": 1.086,
    "args": {
      "External id": 2479427,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218317.969, "dur": 0.923,
    "args": {
      "External id": 2479428,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218320.053, "dur": 0.977,
    "args": {
      "External id": 2479429,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218325.362, "dur": 1.341,
    "args": {
      "External id": 2479430,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218328.532, "dur": 2.122,
    "args": {
      "External id": 2479431,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218332.420, "dur": 1.016,
    "args": {
      "External id": 2479432,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218335.299, "dur": 1.096,
    "args": {
      "External id": 2479433,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218356.297, "dur": 16579.922,
    "args": {
      "External id": 2479434,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218372.489, "dur": 16556.394,
    "args": {
      "External id": 2479435,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218395.973, "dur": 13.078,
    "args": {
      "External id": 2479436,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218412.137, "dur": 16485.890,
    "args": {
      "External id": 2479437,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218414.728, "dur": 16482.735,
    "args": {
      "External id": 2479438,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218420.529, "dur": 6.036,
    "args": {
      "External id": 2479439,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656218428.423, "dur": 16466.116,
    "args": {
      "External id": 2479440,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235139.082, "dur": 35.620,
    "args": {
      "External id": 2479441,"Sequence number": 24740449, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7144
    }
  },
  {
    "ph": "s", "id": 199, "pid": 1336757, "tid": 1336757, "ts": 1514656235139.082,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235161.370, "dur": 8.504,
    "args": {
      "External id": 2479442,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235164.900, "dur": 4.613,
    "args": {
      "External id": 2479443,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235237.899, "dur": 77.689,
    "args": {
      "External id": 2479444,"Record function id": 0, "Ev Idx": 7147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235317.229, "dur": 1068.624,
    "args": {
      "External id": 2479445,"Record function id": 0, "Ev Idx": 7148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235356.386, "dur": 1015.272,
    "args": {
      "External id": 2479446,"Sequence number": 24740450, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7149
    }
  },
  {
    "ph": "s", "id": 198, "pid": 1336757, "tid": 1336757, "ts": 1514656235356.386,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235418.502, "dur": 41.936,
    "args": {
      "External id": 2479447,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235472.121, "dur": 123.489,
    "args": {
      "External id": 2479448,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235608.192, "dur": 39.226,
    "args": {
      "External id": 2479449,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235655.797, "dur": 31.910,
    "args": {
      "External id": 2479450,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235712.076, "dur": 27.403,
    "args": {
      "External id": 2479451,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235756.630, "dur": 13.988,
    "args": {
      "External id": 2479452,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235789.496, "dur": 135.029,
    "args": {
      "External id": 2479453,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235844.868, "dur": 11.918,
    "args": {
      "External id": 2479454,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235850.175, "dur": 5.849,
    "args": {
      "External id": 2479455,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235859.793, "dur": 5.636,
    "args": {
      "External id": 2479456,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235866.882, "dur": 1.335,
    "args": {
      "External id": 2479457,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235870.760, "dur": 4.075,
    "args": {
      "External id": 2479458,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656235935.488, "dur": 44.051,
    "args": {
      "External id": 2479459,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236048.118, "dur": 30.756,
    "args": {
      "External id": 2479460,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236088.063, "dur": 44.494,
    "args": {
      "External id": 2479461,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236141.083, "dur": 34.329,
    "args": {
      "External id": 2479462,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236196.271, "dur": 27.872,
    "args": {
      "External id": 2479463,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236229.509, "dur": 33.125,
    "args": {
      "External id": 2479464,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236282.930, "dur": 23.508,
    "args": {
      "External id": 2479465,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7168
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.14)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236450.125, "dur": 74.903,
    "args": {
      "External id": 2479466,"Record function id": 0, "Ev Idx": 7169
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236616.272, "dur": 46.729,
    "args": {
      "External id": 2479467,"Record function id": 0, "Ev Idx": 7170
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.15)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236672.834, "dur": 18131.686,
    "args": {
      "External id": 2479468,"Record function id": 0, "Ev Idx": 7171
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.15)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236681.556, "dur": 811.533,
    "args": {
      "External id": 2479469,"Record function id": 0, "Ev Idx": 7172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236764.064, "dur": 9.256,
    "args": {
      "External id": 2479470,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236786.343, "dur": 35.957,
    "args": {
      "External id": 2479471,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236791.908, "dur": 2.425,
    "args": {
      "External id": 2479472,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236798.198, "dur": 0.476,
    "args": {
      "External id": 2479473,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236800.188, "dur": 0.566,
    "args": {
      "External id": 2479474,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236802.514, "dur": 0.406,
    "args": {
      "External id": 2479475,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236806.015, "dur": 0.463,
    "args": {
      "External id": 2479476,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236807.776, "dur": 0.305,
    "args": {
      "External id": 2479477,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236809.579, "dur": 3.095,
    "args": {
      "External id": 2479478,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236814.302, "dur": 0.528,
    "args": {
      "External id": 2479479,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236816.117, "dur": 0.398,
    "args": {
      "External id": 2479480,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236833.006, "dur": 41.877,
    "args": {
      "External id": 2479481,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236908.762, "dur": 163.807,
    "args": {
      "External id": 2479482,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236919.195, "dur": 3.736,
    "args": {
      "External id": 2479483,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236927.955, "dur": 10.747,
    "args": {
      "External id": 2479484,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236932.766, "dur": 5.548,
    "args": {
      "External id": 2479485,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236936.523, "dur": 0.634,
    "args": {
      "External id": 2479486,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236945.955, "dur": 28.702,
    "args": {
      "External id": 2479487,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236948.131, "dur": 2.239,
    "args": {
      "External id": 2479488,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236951.879, "dur": 0.304,
    "args": {
      "External id": 2479489,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236953.717, "dur": 0.391,
    "args": {
      "External id": 2479490,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236957.369, "dur": 1.597,
    "args": {
      "External id": 2479491,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236960.246, "dur": 0.638,
    "args": {
      "External id": 2479492,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236962.259, "dur": 0.387,
    "args": {
      "External id": 2479493,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236965.836, "dur": 0.476,
    "args": {
      "External id": 2479494,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236967.532, "dur": 0.273,
    "args": {
      "External id": 2479495,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656236969.164, "dur": 1.680,
    "args": {
      "External id": 2479496,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237033.121, "dur": 29.968,
    "args": {
      "External id": 2479497,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237128.347, "dur": 279.357,
    "args": {
      "External id": 2479498,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237158.705, "dur": 244.462,
    "args": {
      "External id": 2479499,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7202, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237170.284, "dur": 227.656,
    "args": {
      "External id": 2479500,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237428.739, "dur": 2.419,
    "args": {
      "External id": 2479501,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7204, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.15)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237513.424, "dur": 17090.202,
    "args": {
      "External id": 2479502,"Record function id": 0, "Ev Idx": 7205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237636.013, "dur": 6.719,
    "args": {
      "External id": 2479503,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237646.029, "dur": 1.217,
    "args": {
      "External id": 2479504,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237648.853, "dur": 3.110,
    "args": {
      "External id": 2479505,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237653.676, "dur": 0.851,
    "args": {
      "External id": 2479506,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237655.818, "dur": 0.950,
    "args": {
      "External id": 2479507,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237657.965, "dur": 1.494,
    "args": {
      "External id": 2479508,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237662.955, "dur": 0.849,
    "args": {
      "External id": 2479509,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237665.229, "dur": 2.056,
    "args": {
      "External id": 2479510,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237668.631, "dur": 0.854,
    "args": {
      "External id": 2479511,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237670.899, "dur": 0.953,
    "args": {
      "External id": 2479512,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237692.301, "dur": 16869.989,
    "args": {
      "External id": 2479513,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237708.955, "dur": 16845.751,
    "args": {
      "External id": 2479514,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237731.223, "dur": 14.001,
    "args": {
      "External id": 2479515,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237748.220, "dur": 16762.888,
    "args": {
      "External id": 2479516,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237750.701, "dur": 16759.802,
    "args": {
      "External id": 2479517,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237756.071, "dur": 4.892,
    "args": {
      "External id": 2479518,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656237762.445, "dur": 16744.889,
    "args": {
      "External id": 2479519,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656254743.283, "dur": 37.833,
    "args": {
      "External id": 2479520,"Sequence number": 24740451, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7223
    }
  },
  {
    "ph": "s", "id": 197, "pid": 1336757, "tid": 1336757, "ts": 1514656254743.283,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656254767.893, "dur": 8.594,
    "args": {
      "External id": 2479521,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656254771.372, "dur": 4.859,
    "args": {
      "External id": 2479522,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656254843.227, "dur": 77.301,
    "args": {
      "External id": 2479523,"Record function id": 0, "Ev Idx": 7226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656254922.458, "dur": 1057.003,
    "args": {
      "External id": 2479524,"Record function id": 0, "Ev Idx": 7227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656254964.715, "dur": 1001.426,
    "args": {
      "External id": 2479525,"Sequence number": 24740452, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7228
    }
  },
  {
    "ph": "s", "id": 196, "pid": 1336757, "tid": 1336757, "ts": 1514656254964.715,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255066.338, "dur": 43.323,
    "args": {
      "External id": 2479526,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255122.464, "dur": 105.916,
    "args": {
      "External id": 2479527,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255238.189, "dur": 38.174,
    "args": {
      "External id": 2479528,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255284.795, "dur": 30.250,
    "args": {
      "External id": 2479529,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255338.892, "dur": 24.283,
    "args": {
      "External id": 2479530,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255380.321, "dur": 14.003,
    "args": {
      "External id": 2479531,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255411.668, "dur": 146.372,
    "args": {
      "External id": 2479532,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255461.774, "dur": 11.242,
    "args": {
      "External id": 2479533,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255466.833, "dur": 5.430,
    "args": {
      "External id": 2479534,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255475.857, "dur": 6.106,
    "args": {
      "External id": 2479535,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255483.173, "dur": 1.298,
    "args": {
      "External id": 2479536,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255486.782, "dur": 3.829,
    "args": {
      "External id": 2479537,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255570.973, "dur": 48.942,
    "args": {
      "External id": 2479538,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255650.402, "dur": 29.072,
    "args": {
      "External id": 2479539,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255688.055, "dur": 40.104,
    "args": {
      "External id": 2479540,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255735.845, "dur": 34.776,
    "args": {
      "External id": 2479541,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255793.618, "dur": 26.592,
    "args": {
      "External id": 2479542,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255825.262, "dur": 33.625,
    "args": {
      "External id": 2479543,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656255878.142, "dur": 16.406,
    "args": {
      "External id": 2479544,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7247
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.15)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256081.709, "dur": 75.009,
    "args": {
      "External id": 2479545,"Record function id": 0, "Ev Idx": 7248
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256228.528, "dur": 47.217,
    "args": {
      "External id": 2479546,"Record function id": 0, "Ev Idx": 7249
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.16)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256284.951, "dur": 17982.696,
    "args": {
      "External id": 2479547,"Record function id": 0, "Ev Idx": 7250
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.16)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256292.418, "dur": 917.134,
    "args": {
      "External id": 2479548,"Record function id": 0, "Ev Idx": 7251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256374.972, "dur": 9.025,
    "args": {
      "External id": 2479549,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256396.270, "dur": 36.340,
    "args": {
      "External id": 2479550,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256401.726, "dur": 2.417,
    "args": {
      "External id": 2479551,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256408.590, "dur": 0.212,
    "args": {
      "External id": 2479552,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256410.172, "dur": 0.668,
    "args": {
      "External id": 2479553,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256412.483, "dur": 0.620,
    "args": {
      "External id": 2479554,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256416.072, "dur": 0.400,
    "args": {
      "External id": 2479555,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256417.953, "dur": 0.458,
    "args": {
      "External id": 2479556,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256419.768, "dur": 3.024,
    "args": {
      "External id": 2479557,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256424.432, "dur": 0.485,
    "args": {
      "External id": 2479558,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256426.158, "dur": 0.384,
    "args": {
      "External id": 2479559,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256446.167, "dur": 43.282,
    "args": {
      "External id": 2479560,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256522.140, "dur": 136.442,
    "args": {
      "External id": 2479561,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256549.917, "dur": 7.028,
    "args": {
      "External id": 2479562,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256562.694, "dur": 11.176,
    "args": {
      "External id": 2479563,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256567.387, "dur": 6.084,
    "args": {
      "External id": 2479564,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256571.238, "dur": 0.551,
    "args": {
      "External id": 2479565,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256581.715, "dur": 27.789,
    "args": {
      "External id": 2479566,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256583.803, "dur": 0.378,
    "args": {
      "External id": 2479567,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256585.836, "dur": 2.315,
    "args": {
      "External id": 2479568,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256589.474, "dur": 0.359,
    "args": {
      "External id": 2479569,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256591.884, "dur": 1.652,
    "args": {
      "External id": 2479570,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256596.583, "dur": 0.377,
    "args": {
      "External id": 2479571,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256598.381, "dur": 0.431,
    "args": {
      "External id": 2479572,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256600.122, "dur": 0.360,
    "args": {
      "External id": 2479573,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256603.611, "dur": 0.243,
    "args": {
      "External id": 2479574,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256605.034, "dur": 0.190,
    "args": {
      "External id": 2479575,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256621.451, "dur": 28.584,
    "args": {
      "External id": 2479576,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256712.164, "dur": 398.921,
    "args": {
      "External id": 2479577,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256744.092, "dur": 361.152,
    "args": {
      "External id": 2479578,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7281, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656256754.624, "dur": 344.117,
    "args": {
      "External id": 2479579,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257136.726, "dur": 2.437,
    "args": {
      "External id": 2479580,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7283, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.16)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257229.969, "dur": 16847.693,
    "args": {
      "External id": 2479581,"Record function id": 0, "Ev Idx": 7284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257328.760, "dur": 6.340,
    "args": {
      "External id": 2479582,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257338.387, "dur": 1.166,
    "args": {
      "External id": 2479583,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257341.355, "dur": 2.540,
    "args": {
      "External id": 2479584,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257345.449, "dur": 3.164,
    "args": {
      "External id": 2479585,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257350.115, "dur": 1.073,
    "args": {
      "External id": 2479586,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257352.350, "dur": 0.829,
    "args": {
      "External id": 2479587,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257356.935, "dur": 0.768,
    "args": {
      "External id": 2479588,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257359.322, "dur": 2.001,
    "args": {
      "External id": 2479589,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257363.032, "dur": 1.218,
    "args": {
      "External id": 2479590,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257365.765, "dur": 0.930,
    "args": {
      "External id": 2479591,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257386.054, "dur": 16655.236,
    "args": {
      "External id": 2479592,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257401.557, "dur": 16632.637,
    "args": {
      "External id": 2479593,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257425.159, "dur": 13.757,
    "args": {
      "External id": 2479594,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257442.012, "dur": 16538.779,
    "args": {
      "External id": 2479595,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257444.766, "dur": 16535.508,
    "args": {
      "External id": 2479596,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257450.521, "dur": 5.242,
    "args": {
      "External id": 2479597,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656257457.574, "dur": 16519.668,
    "args": {
      "External id": 2479598,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274209.754, "dur": 35.124,
    "args": {
      "External id": 2479599,"Sequence number": 24740453, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7302
    }
  },
  {
    "ph": "s", "id": 195, "pid": 1336757, "tid": 1336757, "ts": 1514656274209.754,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274232.370, "dur": 7.557,
    "args": {
      "External id": 2479600,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274235.337, "dur": 4.344,
    "args": {
      "External id": 2479601,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274304.575, "dur": 79.283,
    "args": {
      "External id": 2479602,"Record function id": 0, "Ev Idx": 7305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274385.457, "dur": 1112.803,
    "args": {
      "External id": 2479603,"Record function id": 0, "Ev Idx": 7306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274425.101, "dur": 1059.402,
    "args": {
      "External id": 2479604,"Sequence number": 24740454, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7307
    }
  },
  {
    "ph": "s", "id": 194, "pid": 1336757, "tid": 1336757, "ts": 1514656274425.101,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274489.750, "dur": 57.543,
    "args": {
      "External id": 2479605,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274563.598, "dur": 101.387,
    "args": {
      "External id": 2479606,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274674.683, "dur": 37.658,
    "args": {
      "External id": 2479607,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274720.814, "dur": 31.851,
    "args": {
      "External id": 2479608,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274784.430, "dur": 29.459,
    "args": {
      "External id": 2479609,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274829.780, "dur": 15.238,
    "args": {
      "External id": 2479610,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274862.186, "dur": 168.966,
    "args": {
      "External id": 2479611,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274913.048, "dur": 11.962,
    "args": {
      "External id": 2479612,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274918.422, "dur": 5.729,
    "args": {
      "External id": 2479613,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274927.872, "dur": 5.942,
    "args": {
      "External id": 2479614,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274935.203, "dur": 1.227,
    "args": {
      "External id": 2479615,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656274939.134, "dur": 5.601,
    "args": {
      "External id": 2479616,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275069.262, "dur": 59.650,
    "args": {
      "External id": 2479617,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275164.367, "dur": 31.046,
    "args": {
      "External id": 2479618,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275204.393, "dur": 42.029,
    "args": {
      "External id": 2479619,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275255.082, "dur": 34.328,
    "args": {
      "External id": 2479620,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275310.309, "dur": 27.834,
    "args": {
      "External id": 2479621,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275343.590, "dur": 33.045,
    "args": {
      "External id": 2479622,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275396.619, "dur": 20.727,
    "args": {
      "External id": 2479623,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7326
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.16)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275579.545, "dur": 82.301,
    "args": {
      "External id": 2479624,"Record function id": 0, "Ev Idx": 7327
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275735.550, "dur": 45.860,
    "args": {
      "External id": 2479625,"Record function id": 0, "Ev Idx": 7328
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.17)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275790.648, "dur": 18000.597,
    "args": {
      "External id": 2479626,"Record function id": 0, "Ev Idx": 7329
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.17)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275798.501, "dur": 927.494,
    "args": {
      "External id": 2479627,"Record function id": 0, "Ev Idx": 7330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275881.099, "dur": 8.896,
    "args": {
      "External id": 2479628,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275903.412, "dur": 35.676,
    "args": {
      "External id": 2479629,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275909.014, "dur": 2.235,
    "args": {
      "External id": 2479630,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275915.541, "dur": 0.214,
    "args": {
      "External id": 2479631,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275917.256, "dur": 0.485,
    "args": {
      "External id": 2479632,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275919.062, "dur": 0.426,
    "args": {
      "External id": 2479633,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275922.543, "dur": 0.232,
    "args": {
      "External id": 2479634,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275924.102, "dur": 0.293,
    "args": {
      "External id": 2479635,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275926.199, "dur": 3.709,
    "args": {
      "External id": 2479636,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275931.206, "dur": 0.249,
    "args": {
      "External id": 2479637,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275933.007, "dur": 0.260,
    "args": {
      "External id": 2479638,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656275950.327, "dur": 80.744,
    "args": {
      "External id": 2479639,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276069.135, "dur": 118.776,
    "args": {
      "External id": 2479640,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276080.935, "dur": 5.107,
    "args": {
      "External id": 2479641,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276090.944, "dur": 10.831,
    "args": {
      "External id": 2479642,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276095.371, "dur": 6.011,
    "args": {
      "External id": 2479643,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276099.234, "dur": 0.572,
    "args": {
      "External id": 2479644,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276109.030, "dur": 29.950,
    "args": {
      "External id": 2479645,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276111.188, "dur": 2.320,
    "args": {
      "External id": 2479646,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276115.285, "dur": 0.277,
    "args": {
      "External id": 2479647,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276117.244, "dur": 0.287,
    "args": {
      "External id": 2479648,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276120.946, "dur": 1.116,
    "args": {
      "External id": 2479649,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276123.416, "dur": 0.409,
    "args": {
      "External id": 2479650,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276125.626, "dur": 0.361,
    "args": {
      "External id": 2479651,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276129.445, "dur": 0.273,
    "args": {
      "External id": 2479652,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276131.095, "dur": 0.167,
    "args": {
      "External id": 2479653,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276132.887, "dur": 1.781,
    "args": {
      "External id": 2479654,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276152.927, "dur": 26.750,
    "args": {
      "External id": 2479655,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276244.289, "dur": 386.350,
    "args": {
      "External id": 2479656,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276276.396, "dur": 349.447,
    "args": {
      "External id": 2479657,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7360, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276286.807, "dur": 332.929,
    "args": {
      "External id": 2479658,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276655.332, "dur": 2.657,
    "args": {
      "External id": 2479659,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7362, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.17)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276746.744, "dur": 16848.210,
    "args": {
      "External id": 2479660,"Record function id": 0, "Ev Idx": 7363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276846.265, "dur": 6.011,
    "args": {
      "External id": 2479661,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276855.854, "dur": 1.803,
    "args": {
      "External id": 2479662,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276859.283, "dur": 2.349,
    "args": {
      "External id": 2479663,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276863.333, "dur": 0.820,
    "args": {
      "External id": 2479664,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276865.421, "dur": 1.107,
    "args": {
      "External id": 2479665,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276867.781, "dur": 0.905,
    "args": {
      "External id": 2479666,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276872.001, "dur": 0.750,
    "args": {
      "External id": 2479667,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276874.446, "dur": 1.707,
    "args": {
      "External id": 2479668,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276877.681, "dur": 0.550,
    "args": {
      "External id": 2479669,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276879.778, "dur": 0.876,
    "args": {
      "External id": 2479670,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276899.566, "dur": 16657.797,
    "args": {
      "External id": 2479671,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276915.831, "dur": 16634.475,
    "args": {
      "External id": 2479672,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276942.694, "dur": 14.305,
    "args": {
      "External id": 2479673,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276959.910, "dur": 16547.607,
    "args": {
      "External id": 2479674,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276962.543, "dur": 16544.402,
    "args": {
      "External id": 2479675,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276968.150, "dur": 5.105,
    "args": {
      "External id": 2479676,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656276975.009, "dur": 16529.022,
    "args": {
      "External id": 2479677,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656293732.835, "dur": 34.778,
    "args": {
      "External id": 2479678,"Sequence number": 24740455, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7381
    }
  },
  {
    "ph": "s", "id": 193, "pid": 1336757, "tid": 1336757, "ts": 1514656293732.835,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656293755.428, "dur": 7.787,
    "args": {
      "External id": 2479679,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656293758.799, "dur": 4.247,
    "args": {
      "External id": 2479680,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656293828.907, "dur": 75.452,
    "args": {
      "External id": 2479681,"Record function id": 0, "Ev Idx": 7384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656293905.888, "dur": 1063.338,
    "args": {
      "External id": 2479682,"Record function id": 0, "Ev Idx": 7385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656293945.982, "dur": 1009.564,
    "args": {
      "External id": 2479683,"Sequence number": 24740456, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7386
    }
  },
  {
    "ph": "s", "id": 192, "pid": 1336757, "tid": 1336757, "ts": 1514656293945.982,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294042.112, "dur": 41.804,
    "args": {
      "External id": 2479684,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294097.670, "dur": 102.540,
    "args": {
      "External id": 2479685,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294210.671, "dur": 37.770,
    "args": {
      "External id": 2479686,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294256.932, "dur": 30.701,
    "args": {
      "External id": 2479687,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294312.241, "dur": 24.130,
    "args": {
      "External id": 2479688,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294352.317, "dur": 15.648,
    "args": {
      "External id": 2479689,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294387.875, "dur": 127.554,
    "args": {
      "External id": 2479690,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294438.814, "dur": 11.651,
    "args": {
      "External id": 2479691,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294443.700, "dur": 6.031,
    "args": {
      "External id": 2479692,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294453.440, "dur": 5.618,
    "args": {
      "External id": 2479693,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294460.500, "dur": 1.236,
    "args": {
      "External id": 2479694,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294464.344, "dur": 3.631,
    "args": {
      "External id": 2479695,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294525.498, "dur": 68.403,
    "args": {
      "External id": 2479696,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294626.999, "dur": 27.650,
    "args": {
      "External id": 2479697,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294663.251, "dur": 40.263,
    "args": {
      "External id": 2479698,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294711.312, "dur": 34.129,
    "args": {
      "External id": 2479699,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294769.425, "dur": 29.252,
    "args": {
      "External id": 2479700,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294805.028, "dur": 39.101,
    "args": {
      "External id": 2479701,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656294866.303, "dur": 18.861,
    "args": {
      "External id": 2479702,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7405
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.17)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295069.576, "dur": 77.809,
    "args": {
      "External id": 2479703,"Record function id": 0, "Ev Idx": 7406
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295219.143, "dur": 43.539,
    "args": {
      "External id": 2479704,"Record function id": 0, "Ev Idx": 7407
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.18)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295272.004, "dur": 18001.666,
    "args": {
      "External id": 2479705,"Record function id": 0, "Ev Idx": 7408
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.18)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295281.298, "dur": 914.951,
    "args": {
      "External id": 2479706,"Record function id": 0, "Ev Idx": 7409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295363.414, "dur": 9.016,
    "args": {
      "External id": 2479707,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295385.368, "dur": 37.000,
    "args": {
      "External id": 2479708,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295390.827, "dur": 2.172,
    "args": {
      "External id": 2479709,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295397.714, "dur": 0.432,
    "args": {
      "External id": 2479710,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295399.759, "dur": 0.544,
    "args": {
      "External id": 2479711,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295401.874, "dur": 0.415,
    "args": {
      "External id": 2479712,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295405.840, "dur": 0.184,
    "args": {
      "External id": 2479713,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295407.897, "dur": 0.324,
    "args": {
      "External id": 2479714,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295409.330, "dur": 3.201,
    "args": {
      "External id": 2479715,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295414.179, "dur": 0.388,
    "args": {
      "External id": 2479716,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295416.041, "dur": 0.295,
    "args": {
      "External id": 2479717,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295433.561, "dur": 41.681,
    "args": {
      "External id": 2479718,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295508.443, "dur": 138.305,
    "args": {
      "External id": 2479719,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295522.694, "dur": 21.692,
    "args": {
      "External id": 2479720,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295551.635, "dur": 11.542,
    "args": {
      "External id": 2479721,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295556.204, "dur": 6.478,
    "args": {
      "External id": 2479722,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295560.332, "dur": 0.646,
    "args": {
      "External id": 2479723,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295571.068, "dur": 29.122,
    "args": {
      "External id": 2479724,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295573.787, "dur": 0.341,
    "args": {
      "External id": 2479725,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295576.048, "dur": 2.215,
    "args": {
      "External id": 2479726,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295579.710, "dur": 0.384,
    "args": {
      "External id": 2479727,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295581.720, "dur": 1.568,
    "args": {
      "External id": 2479728,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295586.977, "dur": 0.335,
    "args": {
      "External id": 2479729,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295588.575, "dur": 0.423,
    "args": {
      "External id": 2479730,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295590.346, "dur": 0.360,
    "args": {
      "External id": 2479731,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295593.551, "dur": 0.258,
    "args": {
      "External id": 2479732,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295595.259, "dur": 0.421,
    "args": {
      "External id": 2479733,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295612.442, "dur": 26.446,
    "args": {
      "External id": 2479734,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295702.667, "dur": 394.871,
    "args": {
      "External id": 2479735,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295733.418, "dur": 358.625,
    "args": {
      "External id": 2479736,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7439, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656295743.548, "dur": 341.527,
    "args": {
      "External id": 2479737,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296122.506, "dur": 2.482,
    "args": {
      "External id": 2479738,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7441, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.18)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296217.160, "dur": 16864.048,
    "args": {
      "External id": 2479739,"Record function id": 0, "Ev Idx": 7442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296320.988, "dur": 6.780,
    "args": {
      "External id": 2479740,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296331.288, "dur": 1.447,
    "args": {
      "External id": 2479741,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296334.414, "dur": 2.211,
    "args": {
      "External id": 2479742,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296338.173, "dur": 0.953,
    "args": {
      "External id": 2479743,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296340.403, "dur": 0.919,
    "args": {
      "External id": 2479744,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296342.510, "dur": 0.687,
    "args": {
      "External id": 2479745,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296346.957, "dur": 0.921,
    "args": {
      "External id": 2479746,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296349.421, "dur": 1.505,
    "args": {
      "External id": 2479747,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296352.517, "dur": 0.655,
    "args": {
      "External id": 2479748,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296354.874, "dur": 0.727,
    "args": {
      "External id": 2479749,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296375.790, "dur": 16666.893,
    "args": {
      "External id": 2479750,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296391.446, "dur": 16644.901,
    "args": {
      "External id": 2479751,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296414.005, "dur": 13.844,
    "args": {
      "External id": 2479752,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296430.899, "dur": 16573.605,
    "args": {
      "External id": 2479753,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296433.497, "dur": 16570.241,
    "args": {
      "External id": 2479754,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296438.985, "dur": 5.843,
    "args": {
      "External id": 2479755,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656296446.765, "dur": 16532.568,
    "args": {
      "External id": 2479756,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313214.998, "dur": 34.503,
    "args": {
      "External id": 2479757,"Sequence number": 24740457, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7460
    }
  },
  {
    "ph": "s", "id": 191, "pid": 1336757, "tid": 1336757, "ts": 1514656313214.998,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313236.778, "dur": 7.805,
    "args": {
      "External id": 2479758,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313240.010, "dur": 4.300,
    "args": {
      "External id": 2479759,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313312.121, "dur": 76.309,
    "args": {
      "External id": 2479760,"Record function id": 0, "Ev Idx": 7463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313389.875, "dur": 1060.316,
    "args": {
      "External id": 2479761,"Record function id": 0, "Ev Idx": 7464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313430.055, "dur": 1006.095,
    "args": {
      "External id": 2479762,"Sequence number": 24740458, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7465
    }
  },
  {
    "ph": "s", "id": 190, "pid": 1336757, "tid": 1336757, "ts": 1514656313430.055,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313498.851, "dur": 56.370,
    "args": {
      "External id": 2479763,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313572.287, "dur": 103.502,
    "args": {
      "External id": 2479764,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313686.081, "dur": 37.257,
    "args": {
      "External id": 2479765,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313729.559, "dur": 31.006,
    "args": {
      "External id": 2479766,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313787.055, "dur": 26.982,
    "args": {
      "External id": 2479767,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313828.988, "dur": 17.319,
    "args": {
      "External id": 2479768,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313861.904, "dur": 164.617,
    "args": {
      "External id": 2479769,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313911.254, "dur": 11.322,
    "args": {
      "External id": 2479770,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313916.205, "dur": 5.498,
    "args": {
      "External id": 2479771,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313925.476, "dur": 5.605,
    "args": {
      "External id": 2479772,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313932.273, "dur": 2.802,
    "args": {
      "External id": 2479773,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656313937.534, "dur": 4.027,
    "args": {
      "External id": 2479774,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314039.771, "dur": 51.704,
    "args": {
      "External id": 2479775,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314123.361, "dur": 29.532,
    "args": {
      "External id": 2479776,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314161.360, "dur": 40.631,
    "args": {
      "External id": 2479777,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314208.380, "dur": 34.400,
    "args": {
      "External id": 2479778,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314267.273, "dur": 26.825,
    "args": {
      "External id": 2479779,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314300.001, "dur": 33.470,
    "args": {
      "External id": 2479780,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314350.207, "dur": 19.239,
    "args": {
      "External id": 2479781,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7484
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.18)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314514.729, "dur": 95.634,
    "args": {
      "External id": 2479782,"Record function id": 0, "Ev Idx": 7485
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314686.192, "dur": 44.628,
    "args": {
      "External id": 2479783,"Record function id": 0, "Ev Idx": 7486
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.19)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314740.447, "dur": 17942.282,
    "args": {
      "External id": 2479784,"Record function id": 0, "Ev Idx": 7487
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.19)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314748.199, "dur": 945.493,
    "args": {
      "External id": 2479785,"Record function id": 0, "Ev Idx": 7488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314830.320, "dur": 9.352,
    "args": {
      "External id": 2479786,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314852.891, "dur": 36.469,
    "args": {
      "External id": 2479787,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314858.609, "dur": 2.369,
    "args": {
      "External id": 2479788,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314865.070, "dur": 0.424,
    "args": {
      "External id": 2479789,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314867.063, "dur": 0.460,
    "args": {
      "External id": 2479790,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314869.007, "dur": 0.371,
    "args": {
      "External id": 2479791,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314872.077, "dur": 0.421,
    "args": {
      "External id": 2479792,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314873.924, "dur": 0.229,
    "args": {
      "External id": 2479793,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314876.068, "dur": 3.433,
    "args": {
      "External id": 2479794,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314881.175, "dur": 0.232,
    "args": {
      "External id": 2479795,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314882.896, "dur": 0.374,
    "args": {
      "External id": 2479796,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314901.313, "dur": 42.074,
    "args": {
      "External id": 2479797,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656314975.904, "dur": 160.611,
    "args": {
      "External id": 2479798,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315026.074, "dur": 5.789,
    "args": {
      "External id": 2479799,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315037.542, "dur": 11.297,
    "args": {
      "External id": 2479800,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315042.167, "dur": 6.286,
    "args": {
      "External id": 2479801,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315045.995, "dur": 0.711,
    "args": {
      "External id": 2479802,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315056.468, "dur": 30.882,
    "args": {
      "External id": 2479803,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315059.183, "dur": 2.254,
    "args": {
      "External id": 2479804,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315063.445, "dur": 0.409,
    "args": {
      "External id": 2479805,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315065.381, "dur": 0.280,
    "args": {
      "External id": 2479806,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315069.152, "dur": 1.460,
    "args": {
      "External id": 2479807,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315072.376, "dur": 0.323,
    "args": {
      "External id": 2479808,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315074.079, "dur": 0.270,
    "args": {
      "External id": 2479809,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315077.318, "dur": 0.156,
    "args": {
      "External id": 2479810,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315078.900, "dur": 0.257,
    "args": {
      "External id": 2479811,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315080.627, "dur": 2.219,
    "args": {
      "External id": 2479812,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315100.170, "dur": 28.075,
    "args": {
      "External id": 2479813,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315191.649, "dur": 405.293,
    "args": {
      "External id": 2479814,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315222.085, "dur": 369.436,
    "args": {
      "External id": 2479815,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7518, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315232.582, "dur": 352.461,
    "args": {
      "External id": 2479816,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315621.547, "dur": 2.698,
    "args": {
      "External id": 2479817,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7520, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.19)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315715.215, "dur": 16747.959,
    "args": {
      "External id": 2479818,"Record function id": 0, "Ev Idx": 7521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315817.391, "dur": 6.546,
    "args": {
      "External id": 2479819,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315839.054, "dur": 1.427,
    "args": {
      "External id": 2479820,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315842.265, "dur": 2.238,
    "args": {
      "External id": 2479821,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315847.477, "dur": 0.985,
    "args": {
      "External id": 2479822,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315850.442, "dur": 0.825,
    "args": {
      "External id": 2479823,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315852.585, "dur": 0.784,
    "args": {
      "External id": 2479824,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315855.087, "dur": 0.794,
    "args": {
      "External id": 2479825,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315859.011, "dur": 1.759,
    "args": {
      "External id": 2479826,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315862.378, "dur": 0.628,
    "args": {
      "External id": 2479827,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315864.853, "dur": 0.652,
    "args": {
      "External id": 2479828,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315884.880, "dur": 16534.892,
    "args": {
      "External id": 2479829,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315900.920, "dur": 16510.963,
    "args": {
      "External id": 2479830,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315919.476, "dur": 13.447,
    "args": {
      "External id": 2479831,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315937.489, "dur": 16439.803,
    "args": {
      "External id": 2479832,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315939.918, "dur": 16436.633,
    "args": {
      "External id": 2479833,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315945.880, "dur": 5.783,
    "args": {
      "External id": 2479834,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656315953.584, "dur": 16419.512,
    "args": {
      "External id": 2479835,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656332619.033, "dur": 39.509,
    "args": {
      "External id": 2479836,"Sequence number": 24740459, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7539
    }
  },
  {
    "ph": "s", "id": 189, "pid": 1336757, "tid": 1336757, "ts": 1514656332619.033,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656332643.545, "dur": 9.591,
    "args": {
      "External id": 2479837,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656332647.351, "dur": 5.416,
    "args": {
      "External id": 2479838,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656332721.788, "dur": 77.923,
    "args": {
      "External id": 2479839,"Record function id": 0, "Ev Idx": 7542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656332801.264, "dur": 1053.910,
    "args": {
      "External id": 2479840,"Record function id": 0, "Ev Idx": 7543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656332838.246, "dur": 1003.468,
    "args": {
      "External id": 2479841,"Sequence number": 24740460, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7544
    }
  },
  {
    "ph": "s", "id": 188, "pid": 1336757, "tid": 1336757, "ts": 1514656332838.246,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656332903.832, "dur": 42.428,
    "args": {
      "External id": 2479842,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656332958.540, "dur": 131.276,
    "args": {
      "External id": 2479843,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333103.777, "dur": 39.777,
    "args": {
      "External id": 2479844,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333152.126, "dur": 30.386,
    "args": {
      "External id": 2479845,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333207.760, "dur": 28.250,
    "args": {
      "External id": 2479846,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333255.761, "dur": 15.317,
    "args": {
      "External id": 2479847,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333290.222, "dur": 125.351,
    "args": {
      "External id": 2479848,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333338.091, "dur": 11.605,
    "args": {
      "External id": 2479849,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333343.089, "dur": 5.755,
    "args": {
      "External id": 2479850,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333352.724, "dur": 6.572,
    "args": {
      "External id": 2479851,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333360.747, "dur": 1.281,
    "args": {
      "External id": 2479852,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333364.315, "dur": 4.005,
    "args": {
      "External id": 2479853,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333425.243, "dur": 46.147,
    "args": {
      "External id": 2479854,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333499.722, "dur": 45.322,
    "args": {
      "External id": 2479855,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333557.116, "dur": 45.144,
    "args": {
      "External id": 2479856,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333611.707, "dur": 34.442,
    "args": {
      "External id": 2479857,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333668.468, "dur": 27.056,
    "args": {
      "External id": 2479858,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333701.073, "dur": 32.985,
    "args": {
      "External id": 2479859,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333754.871, "dur": 16.680,
    "args": {
      "External id": 2479860,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7563
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.19)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656333921.073, "dur": 111.320,
    "args": {
      "External id": 2479861,"Record function id": 0, "Ev Idx": 7564
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334109.700, "dur": 47.625,
    "args": {
      "External id": 2479862,"Record function id": 0, "Ev Idx": 7565
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.20)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334167.509, "dur": 18135.781,
    "args": {
      "External id": 2479863,"Record function id": 0, "Ev Idx": 7566
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.20)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334175.879, "dur": 900.129,
    "args": {
      "External id": 2479864,"Record function id": 0, "Ev Idx": 7567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334258.902, "dur": 9.930,
    "args": {
      "External id": 2479865,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334281.961, "dur": 37.335,
    "args": {
      "External id": 2479866,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334287.487, "dur": 2.309,
    "args": {
      "External id": 2479867,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334294.743, "dur": 0.464,
    "args": {
      "External id": 2479868,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334296.768, "dur": 0.323,
    "args": {
      "External id": 2479869,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334298.921, "dur": 0.289,
    "args": {
      "External id": 2479870,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334302.070, "dur": 0.546,
    "args": {
      "External id": 2479871,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334303.923, "dur": 0.305,
    "args": {
      "External id": 2479872,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334305.425, "dur": 3.118,
    "args": {
      "External id": 2479873,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334310.323, "dur": 0.173,
    "args": {
      "External id": 2479874,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334312.024, "dur": 0.381,
    "args": {
      "External id": 2479875,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334330.197, "dur": 42.075,
    "args": {
      "External id": 2479876,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334405.851, "dur": 112.560,
    "args": {
      "External id": 2479877,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334416.699, "dur": 4.224,
    "args": {
      "External id": 2479878,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334425.896, "dur": 10.808,
    "args": {
      "External id": 2479879,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334430.797, "dur": 5.511,
    "args": {
      "External id": 2479880,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334434.463, "dur": 0.530,
    "args": {
      "External id": 2479881,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334443.843, "dur": 31.842,
    "args": {
      "External id": 2479882,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334445.613, "dur": 2.503,
    "args": {
      "External id": 2479883,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334449.738, "dur": 0.311,
    "args": {
      "External id": 2479884,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334454.756, "dur": 0.354,
    "args": {
      "External id": 2479885,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334458.302, "dur": 1.481,
    "args": {
      "External id": 2479886,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334461.004, "dur": 0.395,
    "args": {
      "External id": 2479887,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334462.820, "dur": 2.055,
    "args": {
      "External id": 2479888,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334466.185, "dur": 0.249,
    "args": {
      "External id": 2479889,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334467.778, "dur": 0.391,
    "args": {
      "External id": 2479890,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334471.055, "dur": 0.184,
    "args": {
      "External id": 2479891,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334485.721, "dur": 24.669,
    "args": {
      "External id": 2479892,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334589.488, "dur": 356.217,
    "args": {
      "External id": 2479893,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334620.877, "dur": 320.168,
    "args": {
      "External id": 2479894,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7597, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334631.056, "dur": 304.534,
    "args": {
      "External id": 2479895,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656334966.803, "dur": 2.742,
    "args": {
      "External id": 2479896,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7599, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.20)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335098.720, "dur": 17007.323,
    "args": {
      "External id": 2479897,"Record function id": 0, "Ev Idx": 7600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335202.031, "dur": 6.641,
    "args": {
      "External id": 2479898,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335212.112, "dur": 1.088,
    "args": {
      "External id": 2479899,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335214.897, "dur": 2.464,
    "args": {
      "External id": 2479900,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335218.891, "dur": 0.817,
    "args": {
      "External id": 2479901,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335221.142, "dur": 0.891,
    "args": {
      "External id": 2479902,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335223.133, "dur": 0.891,
    "args": {
      "External id": 2479903,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335227.378, "dur": 1.059,
    "args": {
      "External id": 2479904,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335229.893, "dur": 1.746,
    "args": {
      "External id": 2479905,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335233.273, "dur": 0.855,
    "args": {
      "External id": 2479906,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335235.442, "dur": 0.878,
    "args": {
      "External id": 2479907,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335256.410, "dur": 16810.711,
    "args": {
      "External id": 2479908,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335272.023, "dur": 16788.383,
    "args": {
      "External id": 2479909,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335295.737, "dur": 13.826,
    "args": {
      "External id": 2479910,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335312.545, "dur": 16716.174,
    "args": {
      "External id": 2479911,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335315.148, "dur": 16712.850,
    "args": {
      "External id": 2479912,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335320.823, "dur": 5.295,
    "args": {
      "External id": 2479913,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656335327.854, "dur": 16697.146,
    "args": {
      "External id": 2479914,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352240.128, "dur": 39.269,
    "args": {
      "External id": 2479915,"Sequence number": 24740461, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7618
    }
  },
  {
    "ph": "s", "id": 187, "pid": 1336757, "tid": 1336757, "ts": 1514656352240.128,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352265.575, "dur": 8.673,
    "args": {
      "External id": 2479916,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352269.632, "dur": 4.383,
    "args": {
      "External id": 2479917,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352340.165, "dur": 75.058,
    "args": {
      "External id": 2479918,"Record function id": 0, "Ev Idx": 7621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352416.596, "dur": 1059.726,
    "args": {
      "External id": 2479919,"Record function id": 0, "Ev Idx": 7622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352456.538, "dur": 1005.985,
    "args": {
      "External id": 2479920,"Sequence number": 24740462, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7623
    }
  },
  {
    "ph": "s", "id": 186, "pid": 1336757, "tid": 1336757, "ts": 1514656352456.538,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352518.391, "dur": 58.636,
    "args": {
      "External id": 2479921,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352592.086, "dur": 105.480,
    "args": {
      "External id": 2479922,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352708.887, "dur": 36.829,
    "args": {
      "External id": 2479923,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352752.594, "dur": 30.931,
    "args": {
      "External id": 2479924,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352811.982, "dur": 26.309,
    "args": {
      "External id": 2479925,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352853.245, "dur": 14.082,
    "args": {
      "External id": 2479926,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352885.375, "dur": 165.630,
    "args": {
      "External id": 2479927,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352934.940, "dur": 11.876,
    "args": {
      "External id": 2479928,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352940.250, "dur": 5.698,
    "args": {
      "External id": 2479929,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352949.975, "dur": 5.300,
    "args": {
      "External id": 2479930,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352956.706, "dur": 1.161,
    "args": {
      "External id": 2479931,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656352961.859, "dur": 3.755,
    "args": {
      "External id": 2479932,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353064.114, "dur": 52.825,
    "args": {
      "External id": 2479933,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353148.477, "dur": 30.142,
    "args": {
      "External id": 2479934,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353188.405, "dur": 40.903,
    "args": {
      "External id": 2479935,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353236.699, "dur": 34.005,
    "args": {
      "External id": 2479936,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353293.205, "dur": 27.752,
    "args": {
      "External id": 2479937,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353326.629, "dur": 32.413,
    "args": {
      "External id": 2479938,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353376.230, "dur": 18.235,
    "args": {
      "External id": 2479939,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7642
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.20)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353555.342, "dur": 74.841,
    "args": {
      "External id": 2479940,"Record function id": 0, "Ev Idx": 7643
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353702.375, "dur": 44.203,
    "args": {
      "External id": 2479941,"Record function id": 0, "Ev Idx": 7644
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.21)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353756.267, "dur": 17961.752,
    "args": {
      "External id": 2479942,"Record function id": 0, "Ev Idx": 7645
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.21)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353766.188, "dur": 961.406,
    "args": {
      "External id": 2479943,"Record function id": 0, "Ev Idx": 7646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353845.780, "dur": 8.767,
    "args": {
      "External id": 2479944,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353867.917, "dur": 35.734,
    "args": {
      "External id": 2479945,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353873.292, "dur": 2.408,
    "args": {
      "External id": 2479946,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353880.533, "dur": 0.195,
    "args": {
      "External id": 2479947,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353882.236, "dur": 0.447,
    "args": {
      "External id": 2479948,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353884.195, "dur": 0.383,
    "args": {
      "External id": 2479949,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353887.864, "dur": 0.385,
    "args": {
      "External id": 2479950,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353890.057, "dur": 0.303,
    "args": {
      "External id": 2479951,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353891.603, "dur": 2.781,
    "args": {
      "External id": 2479952,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353895.572, "dur": 0.216,
    "args": {
      "External id": 2479953,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353897.275, "dur": 0.150,
    "args": {
      "External id": 2479954,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656353914.287, "dur": 41.642,
    "args": {
      "External id": 2479955,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354028.451, "dur": 121.064,
    "args": {
      "External id": 2479956,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354040.749, "dur": 5.592,
    "args": {
      "External id": 2479957,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354051.633, "dur": 11.666,
    "args": {
      "External id": 2479958,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354056.183, "dur": 6.726,
    "args": {
      "External id": 2479959,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354060.467, "dur": 0.635,
    "args": {
      "External id": 2479960,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354070.548, "dur": 30.091,
    "args": {
      "External id": 2479961,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354073.000, "dur": 2.577,
    "args": {
      "External id": 2479962,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354077.157, "dur": 0.303,
    "args": {
      "External id": 2479963,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354078.714, "dur": 0.479,
    "args": {
      "External id": 2479964,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354081.958, "dur": 1.454,
    "args": {
      "External id": 2479965,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354084.532, "dur": 0.318,
    "args": {
      "External id": 2479966,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354086.354, "dur": 0.250,
    "args": {
      "External id": 2479967,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354089.749, "dur": 0.420,
    "args": {
      "External id": 2479968,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354091.323, "dur": 0.440,
    "args": {
      "External id": 2479969,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354093.373, "dur": 2.182,
    "args": {
      "External id": 2479970,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354113.178, "dur": 28.249,
    "args": {
      "External id": 2479971,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354214.002, "dur": 409.744,
    "args": {
      "External id": 2479972,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354247.409, "dur": 369.524,
    "args": {
      "External id": 2479973,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7676, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354257.844, "dur": 352.183,
    "args": {
      "External id": 2479974,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354650.911, "dur": 2.735,
    "args": {
      "External id": 2479975,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7678, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.21)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354749.947, "dur": 16760.429,
    "args": {
      "External id": 2479976,"Record function id": 0, "Ev Idx": 7679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354851.148, "dur": 6.326,
    "args": {
      "External id": 2479977,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354860.600, "dur": 1.248,
    "args": {
      "External id": 2479978,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354863.622, "dur": 2.800,
    "args": {
      "External id": 2479979,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354867.936, "dur": 1.095,
    "args": {
      "External id": 2479980,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354870.295, "dur": 0.872,
    "args": {
      "External id": 2479981,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354872.480, "dur": 0.902,
    "args": {
      "External id": 2479982,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354876.667, "dur": 1.106,
    "args": {
      "External id": 2479983,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354879.655, "dur": 2.033,
    "args": {
      "External id": 2479984,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354883.147, "dur": 1.047,
    "args": {
      "External id": 2479985,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354885.830, "dur": 0.850,
    "args": {
      "External id": 2479986,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354905.972, "dur": 16565.671,
    "args": {
      "External id": 2479987,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354921.368, "dur": 16543.572,
    "args": {
      "External id": 2479988,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354942.511, "dur": 14.346,
    "args": {
      "External id": 2479989,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354959.731, "dur": 16473.261,
    "args": {
      "External id": 2479990,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354962.158, "dur": 16470.201,
    "args": {
      "External id": 2479991,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354967.874, "dur": 5.525,
    "args": {
      "External id": 2479992,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656354974.962, "dur": 16454.485,
    "args": {
      "External id": 2479993,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656371657.831, "dur": 35.539,
    "args": {
      "External id": 2479994,"Sequence number": 24740463, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7697
    }
  },
  {
    "ph": "s", "id": 185, "pid": 1336757, "tid": 1336757, "ts": 1514656371657.831,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656371680.308, "dur": 8.280,
    "args": {
      "External id": 2479995,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656371683.556, "dur": 4.678,
    "args": {
      "External id": 2479996,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656371756.108, "dur": 75.779,
    "args": {
      "External id": 2479997,"Record function id": 0, "Ev Idx": 7700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656371833.581, "dur": 1055.239,
    "args": {
      "External id": 2479998,"Record function id": 0, "Ev Idx": 7701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656371872.233, "dur": 1003.374,
    "args": {
      "External id": 2479999,"Sequence number": 24740464, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7702
    }
  },
  {
    "ph": "s", "id": 184, "pid": 1336757, "tid": 1336757, "ts": 1514656371872.233,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656371943.170, "dur": 71.452,
    "args": {
      "External id": 2480000,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372030.803, "dur": 96.199,
    "args": {
      "External id": 2480001,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372137.143, "dur": 35.849,
    "args": {
      "External id": 2480002,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372181.493, "dur": 30.330,
    "args": {
      "External id": 2480003,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372237.881, "dur": 26.213,
    "args": {
      "External id": 2480004,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372282.279, "dur": 15.060,
    "args": {
      "External id": 2480005,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372316.306, "dur": 128.606,
    "args": {
      "External id": 2480006,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372364.889, "dur": 11.474,
    "args": {
      "External id": 2480007,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372370.209, "dur": 5.406,
    "args": {
      "External id": 2480008,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372379.261, "dur": 6.031,
    "args": {
      "External id": 2480009,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372386.719, "dur": 1.056,
    "args": {
      "External id": 2480010,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372390.633, "dur": 4.668,
    "args": {
      "External id": 2480011,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372455.795, "dur": 43.152,
    "args": {
      "External id": 2480012,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372544.074, "dur": 33.365,
    "args": {
      "External id": 2480013,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372589.409, "dur": 44.184,
    "args": {
      "External id": 2480014,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372642.297, "dur": 34.516,
    "args": {
      "External id": 2480015,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372700.239, "dur": 29.649,
    "args": {
      "External id": 2480016,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372735.178, "dur": 33.472,
    "args": {
      "External id": 2480017,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372787.797, "dur": 17.317,
    "args": {
      "External id": 2480018,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7721
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.21)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656372951.685, "dur": 113.786,
    "args": {
      "External id": 2480019,"Record function id": 0, "Ev Idx": 7722
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373143.651, "dur": 45.031,
    "args": {
      "External id": 2480020,"Record function id": 0, "Ev Idx": 7723
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.22)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373197.968, "dur": 18081.810,
    "args": {
      "External id": 2480021,"Record function id": 0, "Ev Idx": 7724
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.22)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373206.074, "dur": 909.383,
    "args": {
      "External id": 2480022,"Record function id": 0, "Ev Idx": 7725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373286.864, "dur": 9.137,
    "args": {
      "External id": 2480023,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373309.131, "dur": 36.508,
    "args": {
      "External id": 2480024,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373314.996, "dur": 2.553,
    "args": {
      "External id": 2480025,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373321.870, "dur": 0.358,
    "args": {
      "External id": 2480026,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373323.703, "dur": 0.208,
    "args": {
      "External id": 2480027,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373325.371, "dur": 0.406,
    "args": {
      "External id": 2480028,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373328.501, "dur": 0.390,
    "args": {
      "External id": 2480029,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373330.272, "dur": 0.425,
    "args": {
      "External id": 2480030,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373332.002, "dur": 3.846,
    "args": {
      "External id": 2480031,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373337.036, "dur": 0.568,
    "args": {
      "External id": 2480032,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373338.844, "dur": 0.309,
    "args": {
      "External id": 2480033,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373356.866, "dur": 43.089,
    "args": {
      "External id": 2480034,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373433.077, "dur": 132.455,
    "args": {
      "External id": 2480035,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373443.849, "dur": 4.108,
    "args": {
      "External id": 2480036,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373453.097, "dur": 10.379,
    "args": {
      "External id": 2480037,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373457.672, "dur": 5.396,
    "args": {
      "External id": 2480038,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373461.301, "dur": 0.434,
    "args": {
      "External id": 2480039,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373470.541, "dur": 31.677,
    "args": {
      "External id": 2480040,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373472.676, "dur": 2.138,
    "args": {
      "External id": 2480041,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373476.481, "dur": 0.383,
    "args": {
      "External id": 2480042,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373478.162, "dur": 0.523,
    "args": {
      "External id": 2480043,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373481.649, "dur": 1.416,
    "args": {
      "External id": 2480044,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373484.492, "dur": 0.357,
    "args": {
      "External id": 2480045,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373486.052, "dur": 0.356,
    "args": {
      "External id": 2480046,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373489.331, "dur": 0.270,
    "args": {
      "External id": 2480047,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373491.000, "dur": 0.437,
    "args": {
      "External id": 2480048,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373496.511, "dur": 1.813,
    "args": {
      "External id": 2480049,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373511.588, "dur": 43.402,
    "args": {
      "External id": 2480050,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373623.802, "dur": 391.954,
    "args": {
      "External id": 2480051,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373654.822, "dur": 323.035,
    "args": {
      "External id": 2480052,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7755, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656373665.359, "dur": 306.903,
    "args": {
      "External id": 2480053,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374042.013, "dur": 3.463,
    "args": {
      "External id": 2480054,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7757, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.22)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374136.576, "dur": 16954.534,
    "args": {
      "External id": 2480055,"Record function id": 0, "Ev Idx": 7758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374233.063, "dur": 6.797,
    "args": {
      "External id": 2480056,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374242.997, "dur": 1.427,
    "args": {
      "External id": 2480057,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374246.496, "dur": 2.115,
    "args": {
      "External id": 2480058,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374250.250, "dur": 1.013,
    "args": {
      "External id": 2480059,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374252.674, "dur": 1.036,
    "args": {
      "External id": 2480060,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374255.173, "dur": 0.855,
    "args": {
      "External id": 2480061,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374259.990, "dur": 0.949,
    "args": {
      "External id": 2480062,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374262.903, "dur": 2.197,
    "args": {
      "External id": 2480063,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374266.651, "dur": 0.683,
    "args": {
      "External id": 2480064,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374268.819, "dur": 0.661,
    "args": {
      "External id": 2480065,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374289.356, "dur": 16764.306,
    "args": {
      "External id": 2480066,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374310.880, "dur": 16735.819,
    "args": {
      "External id": 2480067,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374329.799, "dur": 13.074,
    "args": {
      "External id": 2480068,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374345.902, "dur": 16668.800,
    "args": {
      "External id": 2480069,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374348.064, "dur": 16666.123,
    "args": {
      "External id": 2480070,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374354.118, "dur": 6.120,
    "args": {
      "External id": 2480071,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656374361.931, "dur": 16649.203,
    "args": {
      "External id": 2480072,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391221.605, "dur": 35.117,
    "args": {
      "External id": 2480073,"Sequence number": 24740465, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7776
    }
  },
  {
    "ph": "s", "id": 183, "pid": 1336757, "tid": 1336757, "ts": 1514656391221.605,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391244.443, "dur": 7.488,
    "args": {
      "External id": 2480074,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391247.436, "dur": 4.320,
    "args": {
      "External id": 2480075,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391315.973, "dur": 76.363,
    "args": {
      "External id": 2480076,"Record function id": 0, "Ev Idx": 7779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391393.735, "dur": 1067.010,
    "args": {
      "External id": 2480077,"Record function id": 0, "Ev Idx": 7780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391432.303, "dur": 1014.987,
    "args": {
      "External id": 2480078,"Sequence number": 24740466, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7781
    }
  },
  {
    "ph": "s", "id": 182, "pid": 1336757, "tid": 1336757, "ts": 1514656391432.303,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391495.948, "dur": 56.276,
    "args": {
      "External id": 2480079,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391568.542, "dur": 102.931,
    "args": {
      "External id": 2480080,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391681.894, "dur": 38.817,
    "args": {
      "External id": 2480081,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391728.002, "dur": 30.844,
    "args": {
      "External id": 2480082,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391784.528, "dur": 28.662,
    "args": {
      "External id": 2480083,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391830.540, "dur": 14.283,
    "args": {
      "External id": 2480084,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391863.077, "dur": 165.599,
    "args": {
      "External id": 2480085,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391915.043, "dur": 12.147,
    "args": {
      "External id": 2480086,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391920.471, "dur": 5.962,
    "args": {
      "External id": 2480087,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391930.463, "dur": 5.068,
    "args": {
      "External id": 2480088,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391936.776, "dur": 1.417,
    "args": {
      "External id": 2480089,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656391940.527, "dur": 3.611,
    "args": {
      "External id": 2480090,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392041.906, "dur": 54.574,
    "args": {
      "External id": 2480091,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392130.017, "dur": 29.925,
    "args": {
      "External id": 2480092,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392168.645, "dur": 40.987,
    "args": {
      "External id": 2480093,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392217.548, "dur": 34.283,
    "args": {
      "External id": 2480094,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392274.079, "dur": 29.322,
    "args": {
      "External id": 2480095,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392309.194, "dur": 32.799,
    "args": {
      "External id": 2480096,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392360.225, "dur": 17.421,
    "args": {
      "External id": 2480097,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7800
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.22)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392525.048, "dur": 93.730,
    "args": {
      "External id": 2480098,"Record function id": 0, "Ev Idx": 7801
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392694.646, "dur": 45.176,
    "args": {
      "External id": 2480099,"Record function id": 0, "Ev Idx": 7802
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.23)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392749.397, "dur": 18087.340,
    "args": {
      "External id": 2480100,"Record function id": 0, "Ev Idx": 7803
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.23)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392757.991, "dur": 964.417,
    "args": {
      "External id": 2480101,"Record function id": 0, "Ev Idx": 7804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392839.365, "dur": 8.744,
    "args": {
      "External id": 2480102,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392860.950, "dur": 37.265,
    "args": {
      "External id": 2480103,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392866.674, "dur": 2.282,
    "args": {
      "External id": 2480104,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392873.488, "dur": 0.353,
    "args": {
      "External id": 2480105,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392875.396, "dur": 0.465,
    "args": {
      "External id": 2480106,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392877.408, "dur": 0.411,
    "args": {
      "External id": 2480107,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392881.055, "dur": 0.337,
    "args": {
      "External id": 2480108,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392882.883, "dur": 0.232,
    "args": {
      "External id": 2480109,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392884.635, "dur": 3.059,
    "args": {
      "External id": 2480110,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392889.426, "dur": 0.353,
    "args": {
      "External id": 2480111,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392891.115, "dur": 0.540,
    "args": {
      "External id": 2480112,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656392909.545, "dur": 41.789,
    "args": {
      "External id": 2480113,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393023.578, "dur": 152.085,
    "args": {
      "External id": 2480114,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393035.994, "dur": 5.797,
    "args": {
      "External id": 2480115,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393047.329, "dur": 10.628,
    "args": {
      "External id": 2480116,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393051.785, "dur": 5.798,
    "args": {
      "External id": 2480117,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393055.611, "dur": 0.504,
    "args": {
      "External id": 2480118,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393065.938, "dur": 32.429,
    "args": {
      "External id": 2480119,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393069.151, "dur": 2.000,
    "args": {
      "External id": 2480120,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393073.684, "dur": 0.390,
    "args": {
      "External id": 2480121,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393075.799, "dur": 0.489,
    "args": {
      "External id": 2480122,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393079.286, "dur": 1.435,
    "args": {
      "External id": 2480123,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393082.084, "dur": 0.418,
    "args": {
      "External id": 2480124,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393084.000, "dur": 0.370,
    "args": {
      "External id": 2480125,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393087.538, "dur": 0.511,
    "args": {
      "External id": 2480126,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393089.533, "dur": 0.257,
    "args": {
      "External id": 2480127,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393091.244, "dur": 2.261,
    "args": {
      "External id": 2480128,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393137.438, "dur": 29.756,
    "args": {
      "External id": 2480129,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393231.077, "dur": 395.937,
    "args": {
      "External id": 2480130,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393261.458, "dur": 360.331,
    "args": {
      "External id": 2480131,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7834, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393272.885, "dur": 342.536,
    "args": {
      "External id": 2480132,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393650.236, "dur": 2.529,
    "args": {
      "External id": 2480133,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7836, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.23)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393743.381, "dur": 16897.109,
    "args": {
      "External id": 2480134,"Record function id": 0, "Ev Idx": 7837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393844.464, "dur": 6.295,
    "args": {
      "External id": 2480135,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393853.842, "dur": 1.050,
    "args": {
      "External id": 2480136,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393856.681, "dur": 2.445,
    "args": {
      "External id": 2480137,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393860.737, "dur": 0.760,
    "args": {
      "External id": 2480138,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393862.827, "dur": 1.038,
    "args": {
      "External id": 2480139,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393865.146, "dur": 0.712,
    "args": {
      "External id": 2480140,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393868.867, "dur": 0.670,
    "args": {
      "External id": 2480141,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393871.088, "dur": 2.725,
    "args": {
      "External id": 2480142,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393875.604, "dur": 0.849,
    "args": {
      "External id": 2480143,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393877.793, "dur": 0.954,
    "args": {
      "External id": 2480144,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393898.413, "dur": 16704.790,
    "args": {
      "External id": 2480145,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393913.680, "dur": 16682.614,
    "args": {
      "External id": 2480146,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393935.918, "dur": 13.627,
    "args": {
      "External id": 2480147,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393952.445, "dur": 16611.612,
    "args": {
      "External id": 2480148,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393955.046, "dur": 16608.415,
    "args": {
      "External id": 2480149,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393961.128, "dur": 5.344,
    "args": {
      "External id": 2480150,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656393968.292, "dur": 16592.368,
    "args": {
      "External id": 2480151,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656410774.382, "dur": 38.785,
    "args": {
      "External id": 2480152,"Sequence number": 24740467, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7855
    }
  },
  {
    "ph": "s", "id": 181, "pid": 1336757, "tid": 1336757, "ts": 1514656410774.382,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656410800.701, "dur": 7.721,
    "args": {
      "External id": 2480153,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656410803.803, "dur": 4.387,
    "args": {
      "External id": 2480154,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656410874.260, "dur": 75.337,
    "args": {
      "External id": 2480155,"Record function id": 0, "Ev Idx": 7858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656410951.138, "dur": 1092.400,
    "args": {
      "External id": 2480156,"Record function id": 0, "Ev Idx": 7859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411025.156, "dur": 1004.167,
    "args": {
      "External id": 2480157,"Sequence number": 24740468, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7860
    }
  },
  {
    "ph": "s", "id": 180, "pid": 1336757, "tid": 1336757, "ts": 1514656411025.156,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411093.537, "dur": 45.036,
    "args": {
      "External id": 2480158,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411151.084, "dur": 102.969,
    "args": {
      "External id": 2480159,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411263.223, "dur": 38.852,
    "args": {
      "External id": 2480160,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411310.680, "dur": 31.052,
    "args": {
      "External id": 2480161,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411365.596, "dur": 25.287,
    "args": {
      "External id": 2480162,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411409.352, "dur": 15.220,
    "args": {
      "External id": 2480163,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411442.033, "dur": 147.581,
    "args": {
      "External id": 2480164,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411492.200, "dur": 11.816,
    "args": {
      "External id": 2480165,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411497.248, "dur": 5.943,
    "args": {
      "External id": 2480166,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411506.635, "dur": 5.712,
    "args": {
      "External id": 2480167,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411513.611, "dur": 1.247,
    "args": {
      "External id": 2480168,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411517.306, "dur": 3.586,
    "args": {
      "External id": 2480169,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411602.416, "dur": 48.326,
    "args": {
      "External id": 2480170,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411679.381, "dur": 29.877,
    "args": {
      "External id": 2480171,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411717.805, "dur": 39.987,
    "args": {
      "External id": 2480172,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411766.662, "dur": 33.934,
    "args": {
      "External id": 2480173,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411821.929, "dur": 26.961,
    "args": {
      "External id": 2480174,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411854.375, "dur": 32.466,
    "args": {
      "External id": 2480175,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656411905.190, "dur": 16.798,
    "args": {
      "External id": 2480176,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7879
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.23)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412108.987, "dur": 73.993,
    "args": {
      "External id": 2480177,"Record function id": 0, "Ev Idx": 7880
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412252.899, "dur": 44.325,
    "args": {
      "External id": 2480178,"Record function id": 0, "Ev Idx": 7881
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.24)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412307.651, "dur": 18297.894,
    "args": {
      "External id": 2480179,"Record function id": 0, "Ev Idx": 7882
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.24)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412316.159, "dur": 911.076,
    "args": {
      "External id": 2480180,"Record function id": 0, "Ev Idx": 7883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412396.985, "dur": 8.883,
    "args": {
      "External id": 2480181,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412418.311, "dur": 34.816,
    "args": {
      "External id": 2480182,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412424.450, "dur": 2.182,
    "args": {
      "External id": 2480183,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412430.546, "dur": 0.322,
    "args": {
      "External id": 2480184,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412431.930, "dur": 0.446,
    "args": {
      "External id": 2480185,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412433.295, "dur": 0.421,
    "args": {
      "External id": 2480186,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412436.613, "dur": 0.337,
    "args": {
      "External id": 2480187,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412438.129, "dur": 0.407,
    "args": {
      "External id": 2480188,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412439.613, "dur": 3.459,
    "args": {
      "External id": 2480189,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412444.507, "dur": 0.323,
    "args": {
      "External id": 2480190,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412445.982, "dur": 0.327,
    "args": {
      "External id": 2480191,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412464.147, "dur": 40.746,
    "args": {
      "External id": 2480192,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412555.617, "dur": 114.201,
    "args": {
      "External id": 2480193,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412567.135, "dur": 5.702,
    "args": {
      "External id": 2480194,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412578.311, "dur": 11.229,
    "args": {
      "External id": 2480195,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412582.875, "dur": 6.284,
    "args": {
      "External id": 2480196,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412586.786, "dur": 0.842,
    "args": {
      "External id": 2480197,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412596.263, "dur": 26.810,
    "args": {
      "External id": 2480198,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412598.588, "dur": 2.098,
    "args": {
      "External id": 2480199,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412602.043, "dur": 0.552,
    "args": {
      "External id": 2480200,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412603.497, "dur": 0.326,
    "args": {
      "External id": 2480201,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412606.738, "dur": 1.570,
    "args": {
      "External id": 2480202,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412609.121, "dur": 0.381,
    "args": {
      "External id": 2480203,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412610.365, "dur": 0.570,
    "args": {
      "External id": 2480204,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412613.620, "dur": 0.166,
    "args": {
      "External id": 2480205,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412614.848, "dur": 0.321,
    "args": {
      "External id": 2480206,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412616.259, "dur": 1.684,
    "args": {
      "External id": 2480207,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412634.454, "dur": 27.480,
    "args": {
      "External id": 2480208,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412724.177, "dur": 403.703,
    "args": {
      "External id": 2480209,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412760.681, "dur": 361.615,
    "args": {
      "External id": 2480210,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7913, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656412771.113, "dur": 345.170,
    "args": {
      "External id": 2480211,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413151.545, "dur": 2.765,
    "args": {
      "External id": 2480212,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7915, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.24)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413248.996, "dur": 17149.380,
    "args": {
      "External id": 2480213,"Record function id": 0, "Ev Idx": 7916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413349.951, "dur": 6.639,
    "args": {
      "External id": 2480214,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413365.294, "dur": 1.096,
    "args": {
      "External id": 2480215,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413368.102, "dur": 2.062,
    "args": {
      "External id": 2480216,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413373.125, "dur": 1.016,
    "args": {
      "External id": 2480217,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413375.774, "dur": 0.969,
    "args": {
      "External id": 2480218,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413378.108, "dur": 0.615,
    "args": {
      "External id": 2480219,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413381.615, "dur": 1.036,
    "args": {
      "External id": 2480220,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413385.500, "dur": 2.002,
    "args": {
      "External id": 2480221,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413389.102, "dur": 0.936,
    "args": {
      "External id": 2480222,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413391.569, "dur": 0.642,
    "args": {
      "External id": 2480223,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 7926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413412.088, "dur": 16946.745,
    "args": {
      "External id": 2480224,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413427.345, "dur": 16923.799,
    "args": {
      "External id": 2480225,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 7928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413446.816, "dur": 14.373,
    "args": {
      "External id": 2480226,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413465.863, "dur": 16854.847,
    "args": {
      "External id": 2480227,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 7930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413468.366, "dur": 16851.801,
    "args": {
      "External id": 2480228,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 7931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413474.726, "dur": 5.643,
    "args": {
      "External id": 2480229,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656413482.159, "dur": 16834.842,
    "args": {
      "External id": 2480230,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 7933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656430544.496, "dur": 35.641,
    "args": {
      "External id": 2480231,"Sequence number": 24740469, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 7934
    }
  },
  {
    "ph": "s", "id": 179, "pid": 1336757, "tid": 1336757, "ts": 1514656430544.496,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656430567.109, "dur": 7.797,
    "args": {
      "External id": 2480232,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 7935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656430570.061, "dur": 4.432,
    "args": {
      "External id": 2480233,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 7936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656430642.526, "dur": 75.906,
    "args": {
      "External id": 2480234,"Record function id": 0, "Ev Idx": 7937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656430719.805, "dur": 1064.652,
    "args": {
      "External id": 2480235,"Record function id": 0, "Ev Idx": 7938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656430759.132, "dur": 1011.780,
    "args": {
      "External id": 2480236,"Sequence number": 24740470, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 7939
    }
  },
  {
    "ph": "s", "id": 178, "pid": 1336757, "tid": 1336757, "ts": 1514656430759.132,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656430824.968, "dur": 43.259,
    "args": {
      "External id": 2480237,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656430880.813, "dur": 139.741,
    "args": {
      "External id": 2480238,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431034.251, "dur": 43.056,
    "args": {
      "External id": 2480239,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431085.792, "dur": 30.150,
    "args": {
      "External id": 2480240,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431140.565, "dur": 25.817,
    "args": {
      "External id": 2480241,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431184.516, "dur": 14.898,
    "args": {
      "External id": 2480242,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 7945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431218.798, "dur": 128.845,
    "args": {
      "External id": 2480243,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 7946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431269.723, "dur": 11.828,
    "args": {
      "External id": 2480244,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 7947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431275.135, "dur": 5.467,
    "args": {
      "External id": 2480245,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431284.177, "dur": 5.745,
    "args": {
      "External id": 2480246,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431291.050, "dur": 1.203,
    "args": {
      "External id": 2480247,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431294.591, "dur": 3.951,
    "args": {
      "External id": 2480248,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431358.252, "dur": 42.421,
    "args": {
      "External id": 2480249,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 7952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431429.226, "dur": 26.666,
    "args": {
      "External id": 2480250,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 7953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431464.736, "dur": 39.838,
    "args": {
      "External id": 2480251,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431512.700, "dur": 51.657,
    "args": {
      "External id": 2480252,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 7955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431598.196, "dur": 25.919,
    "args": {
      "External id": 2480253,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 7956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431632.354, "dur": 36.053,
    "args": {
      "External id": 2480254,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 7957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431687.844, "dur": 18.818,
    "args": {
      "External id": 2480255,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 7958
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.24)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656431847.785, "dur": 76.766,
    "args": {
      "External id": 2480256,"Record function id": 0, "Ev Idx": 7959
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432034.856, "dur": 47.296,
    "args": {
      "External id": 2480257,"Record function id": 0, "Ev Idx": 7960
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.25)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432091.924, "dur": 18070.159,
    "args": {
      "External id": 2480258,"Record function id": 0, "Ev Idx": 7961
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.25)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432098.912, "dur": 932.486,
    "args": {
      "External id": 2480259,"Record function id": 0, "Ev Idx": 7962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432180.034, "dur": 10.306,
    "args": {
      "External id": 2480260,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432203.300, "dur": 33.233,
    "args": {
      "External id": 2480261,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432208.867, "dur": 2.294,
    "args": {
      "External id": 2480262,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432215.114, "dur": 0.213,
    "args": {
      "External id": 2480263,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432216.633, "dur": 0.493,
    "args": {
      "External id": 2480264,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432218.491, "dur": 0.295,
    "args": {
      "External id": 2480265,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432221.159, "dur": 0.318,
    "args": {
      "External id": 2480266,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432222.639, "dur": 0.351,
    "args": {
      "External id": 2480267,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432224.192, "dur": 3.074,
    "args": {
      "External id": 2480268,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432228.209, "dur": 0.194,
    "args": {
      "External id": 2480269,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432229.329, "dur": 0.178,
    "args": {
      "External id": 2480270,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432247.802, "dur": 42.911,
    "args": {
      "External id": 2480271,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432322.740, "dur": 106.549,
    "args": {
      "External id": 2480272,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 7975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432333.507, "dur": 3.862,
    "args": {
      "External id": 2480273,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 7976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432342.194, "dur": 10.153,
    "args": {
      "External id": 2480274,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432346.789, "dur": 5.106,
    "args": {
      "External id": 2480275,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 7978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432350.091, "dur": 0.501,
    "args": {
      "External id": 2480276,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 7979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432359.051, "dur": 26.426,
    "args": {
      "External id": 2480277,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 7980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432361.011, "dur": 2.162,
    "args": {
      "External id": 2480278,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432364.488, "dur": 0.319,
    "args": {
      "External id": 2480279,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432365.879, "dur": 0.148,
    "args": {
      "External id": 2480280,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432368.854, "dur": 1.541,
    "args": {
      "External id": 2480281,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432371.404, "dur": 0.287,
    "args": {
      "External id": 2480282,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432372.953, "dur": 0.399,
    "args": {
      "External id": 2480283,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432375.811, "dur": 0.257,
    "args": {
      "External id": 2480284,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432377.117, "dur": 0.262,
    "args": {
      "External id": 2480285,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432378.358, "dur": 2.248,
    "args": {
      "External id": 2480286,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 7989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432396.850, "dur": 24.519,
    "args": {
      "External id": 2480287,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 7990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432481.921, "dur": 416.982,
    "args": {
      "External id": 2480288,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 7991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432511.749, "dur": 381.980,
    "args": {
      "External id": 2480289,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 7992, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432522.571, "dur": 365.498,
    "args": {
      "External id": 2480290,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 7993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656432922.500, "dur": 2.539,
    "args": {
      "External id": 2480291,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 7994, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.25)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433054.558, "dur": 16887.778,
    "args": {
      "External id": 2480292,"Record function id": 0, "Ev Idx": 7995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433160.110, "dur": 6.967,
    "args": {
      "External id": 2480293,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 7996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433170.419, "dur": 1.240,
    "args": {
      "External id": 2480294,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 7997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433173.272, "dur": 2.053,
    "args": {
      "External id": 2480295,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433177.098, "dur": 0.824,
    "args": {
      "External id": 2480296,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 7999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433179.561, "dur": 0.785,
    "args": {
      "External id": 2480297,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433181.464, "dur": 0.848,
    "args": {
      "External id": 2480298,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433185.398, "dur": 0.824,
    "args": {
      "External id": 2480299,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433187.818, "dur": 1.760,
    "args": {
      "External id": 2480300,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433191.143, "dur": 0.654,
    "args": {
      "External id": 2480301,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433193.349, "dur": 0.808,
    "args": {
      "External id": 2480302,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433213.321, "dur": 16690.315,
    "args": {
      "External id": 2480303,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433229.069, "dur": 16667.718,
    "args": {
      "External id": 2480304,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433254.361, "dur": 12.800,
    "args": {
      "External id": 2480305,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433270.085, "dur": 16595.354,
    "args": {
      "External id": 2480306,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 8009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433272.528, "dur": 16592.335,
    "args": {
      "External id": 2480307,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 8010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433278.009, "dur": 5.420,
    "args": {
      "External id": 2480308,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656433284.998, "dur": 16576.882,
    "args": {
      "External id": 2480309,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 8012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450101.095, "dur": 36.595,
    "args": {
      "External id": 2480310,"Sequence number": 24740471, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 8013
    }
  },
  {
    "ph": "s", "id": 177, "pid": 1336757, "tid": 1336757, "ts": 1514656450101.095,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450124.420, "dur": 8.435,
    "args": {
      "External id": 2480311,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 8014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450127.670, "dur": 4.789,
    "args": {
      "External id": 2480312,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450197.093, "dur": 74.427,
    "args": {
      "External id": 2480313,"Record function id": 0, "Ev Idx": 8016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450273.056, "dur": 1064.111,
    "args": {
      "External id": 2480314,"Record function id": 0, "Ev Idx": 8017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450314.258, "dur": 1008.515,
    "args": {
      "External id": 2480315,"Sequence number": 24740472, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 8018
    }
  },
  {
    "ph": "s", "id": 176, "pid": 1336757, "tid": 1336757, "ts": 1514656450314.258,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450380.172, "dur": 41.250,
    "args": {
      "External id": 2480316,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450433.315, "dur": 113.555,
    "args": {
      "External id": 2480317,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450559.201, "dur": 41.410,
    "args": {
      "External id": 2480318,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450610.213, "dur": 30.898,
    "args": {
      "External id": 2480319,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450667.465, "dur": 25.374,
    "args": {
      "External id": 2480320,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450710.051, "dur": 14.713,
    "args": {
      "External id": 2480321,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450742.163, "dur": 130.435,
    "args": {
      "External id": 2480322,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450795.393, "dur": 12.111,
    "args": {
      "External id": 2480323,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450800.475, "dur": 6.156,
    "args": {
      "External id": 2480324,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450810.157, "dur": 5.640,
    "args": {
      "External id": 2480325,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450816.934, "dur": 1.822,
    "args": {
      "External id": 2480326,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450821.076, "dur": 2.798,
    "args": {
      "External id": 2480327,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450883.712, "dur": 43.729,
    "args": {
      "External id": 2480328,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656450957.808, "dur": 63.079,
    "args": {
      "External id": 2480329,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451032.874, "dur": 45.213,
    "args": {
      "External id": 2480330,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451087.458, "dur": 34.941,
    "args": {
      "External id": 2480331,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451147.048, "dur": 32.131,
    "args": {
      "External id": 2480332,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 8035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451185.282, "dur": 33.266,
    "args": {
      "External id": 2480333,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 8036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451237.000, "dur": 16.254,
    "args": {
      "External id": 2480334,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 8037
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.25)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451402.244, "dur": 76.800,
    "args": {
      "External id": 2480335,"Record function id": 0, "Ev Idx": 8038
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451571.203, "dur": 48.396,
    "args": {
      "External id": 2480336,"Record function id": 0, "Ev Idx": 8039
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.26)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451629.549, "dur": 18094.210,
    "args": {
      "External id": 2480337,"Record function id": 0, "Ev Idx": 8040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.26)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451637.766, "dur": 868.032,
    "args": {
      "External id": 2480338,"Record function id": 0, "Ev Idx": 8041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451720.582, "dur": 9.811,
    "args": {
      "External id": 2480339,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451743.044, "dur": 34.605,
    "args": {
      "External id": 2480340,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451748.543, "dur": 2.297,
    "args": {
      "External id": 2480341,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451754.925, "dur": 0.476,
    "args": {
      "External id": 2480342,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451756.520, "dur": 0.675,
    "args": {
      "External id": 2480343,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451758.584, "dur": 0.700,
    "args": {
      "External id": 2480344,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451762.031, "dur": 0.617,
    "args": {
      "External id": 2480345,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451763.581, "dur": 0.613,
    "args": {
      "External id": 2480346,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451765.273, "dur": 2.387,
    "args": {
      "External id": 2480347,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451768.827, "dur": 0.561,
    "args": {
      "External id": 2480348,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451770.731, "dur": 0.735,
    "args": {
      "External id": 2480349,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451788.103, "dur": 40.943,
    "args": {
      "External id": 2480350,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451862.338, "dur": 112.021,
    "args": {
      "External id": 2480351,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 8054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451872.495, "dur": 4.737,
    "args": {
      "External id": 2480352,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451882.418, "dur": 10.181,
    "args": {
      "External id": 2480353,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451886.707, "dur": 5.491,
    "args": {
      "External id": 2480354,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 8057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451890.130, "dur": 0.760,
    "args": {
      "External id": 2480355,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451898.924, "dur": 32.954,
    "args": {
      "External id": 2480356,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451901.184, "dur": 2.598,
    "args": {
      "External id": 2480357,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451904.932, "dur": 0.728,
    "args": {
      "External id": 2480358,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451909.563, "dur": 2.118,
    "args": {
      "External id": 2480359,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451914.670, "dur": 0.795,
    "args": {
      "External id": 2480360,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451916.745, "dur": 0.477,
    "args": {
      "External id": 2480361,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451918.189, "dur": 2.264,
    "args": {
      "External id": 2480362,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451921.818, "dur": 0.452,
    "args": {
      "External id": 2480363,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451923.356, "dur": 0.600,
    "args": {
      "External id": 2480364,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451927.115, "dur": 0.502,
    "args": {
      "External id": 2480365,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656451942.606, "dur": 23.942,
    "args": {
      "External id": 2480366,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452069.296, "dur": 349.527,
    "args": {
      "External id": 2480367,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 8070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452100.136, "dur": 313.892,
    "args": {
      "External id": 2480368,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 8071, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452110.434, "dur": 295.784,
    "args": {
      "External id": 2480369,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 8072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452438.740, "dur": 2.407,
    "args": {
      "External id": 2480370,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 8073, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.26)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452525.143, "dur": 16991.892,
    "args": {
      "External id": 2480371,"Record function id": 0, "Ev Idx": 8074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452644.367, "dur": 6.794,
    "args": {
      "External id": 2480372,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 8075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452654.536, "dur": 1.676,
    "args": {
      "External id": 2480373,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452663.962, "dur": 1.359,
    "args": {
      "External id": 2480374,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452668.723, "dur": 1.395,
    "args": {
      "External id": 2480375,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452671.633, "dur": 1.065,
    "args": {
      "External id": 2480376,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452674.289, "dur": 1.313,
    "args": {
      "External id": 2480377,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452677.305, "dur": 1.277,
    "args": {
      "External id": 2480378,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452681.261, "dur": 1.934,
    "args": {
      "External id": 2480379,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452684.725, "dur": 1.023,
    "args": {
      "External id": 2480380,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452687.249, "dur": 1.088,
    "args": {
      "External id": 2480381,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452708.876, "dur": 16769.806,
    "args": {
      "External id": 2480382,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452725.100, "dur": 16746.545,
    "args": {
      "External id": 2480383,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452744.068, "dur": 13.643,
    "args": {
      "External id": 2480384,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452762.251, "dur": 16677.246,
    "args": {
      "External id": 2480385,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 8088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452764.799, "dur": 16674.125,
    "args": {
      "External id": 2480386,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 8089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452770.318, "dur": 5.441,
    "args": {
      "External id": 2480387,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656452777.343, "dur": 16658.623,
    "args": {
      "External id": 2480388,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 8091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656469662.680, "dur": 36.325,
    "args": {
      "External id": 2480389,"Sequence number": 24740473, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 8092
    }
  },
  {
    "ph": "s", "id": 175, "pid": 1336757, "tid": 1336757, "ts": 1514656469662.680,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656469685.617, "dur": 8.512,
    "args": {
      "External id": 2480390,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 8093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656469688.824, "dur": 4.989,
    "args": {
      "External id": 2480391,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656469761.511, "dur": 75.649,
    "args": {
      "External id": 2480392,"Record function id": 0, "Ev Idx": 8095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656469838.885, "dur": 1026.112,
    "args": {
      "External id": 2480393,"Record function id": 0, "Ev Idx": 8096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656469876.275, "dur": 975.581,
    "args": {
      "External id": 2480394,"Sequence number": 24740474, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 8097
    }
  },
  {
    "ph": "s", "id": 174, "pid": 1336757, "tid": 1336757, "ts": 1514656469876.275,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656469939.812, "dur": 69.901,
    "args": {
      "External id": 2480395,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470026.377, "dur": 94.933,
    "args": {
      "External id": 2480396,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470130.755, "dur": 36.258,
    "args": {
      "External id": 2480397,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470175.104, "dur": 30.403,
    "args": {
      "External id": 2480398,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470229.544, "dur": 26.075,
    "args": {
      "External id": 2480399,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470273.720, "dur": 14.665,
    "args": {
      "External id": 2480400,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470306.154, "dur": 126.390,
    "args": {
      "External id": 2480401,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470356.499, "dur": 12.346,
    "args": {
      "External id": 2480402,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470361.824, "dur": 6.230,
    "args": {
      "External id": 2480403,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470371.736, "dur": 4.932,
    "args": {
      "External id": 2480404,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470377.916, "dur": 1.577,
    "args": {
      "External id": 2480405,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470381.896, "dur": 3.045,
    "args": {
      "External id": 2480406,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470443.190, "dur": 42.555,
    "args": {
      "External id": 2480407,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470514.592, "dur": 45.009,
    "args": {
      "External id": 2480408,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470570.779, "dur": 43.442,
    "args": {
      "External id": 2480409,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470623.357, "dur": 34.529,
    "args": {
      "External id": 2480410,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470679.623, "dur": 26.777,
    "args": {
      "External id": 2480411,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 8114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470711.818, "dur": 33.834,
    "args": {
      "External id": 2480412,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 8115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470764.994, "dur": 17.049,
    "args": {
      "External id": 2480413,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 8116
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.26)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656470928.273, "dur": 114.759,
    "args": {
      "External id": 2480414,"Record function id": 0, "Ev Idx": 8117
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::cast_forward_inputs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471121.034, "dur": 47.506,
    "args": {
      "External id": 2480415,"Record function id": 0, "Ev Idx": 8118
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::pre_forward (model.layers.27)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471177.407, "dur": 18241.830,
    "args": {
      "External id": 2480416,"Record function id": 0, "Ev Idx": 8119
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather (model.layers.27)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471185.056, "dur": 910.136,
    "args": {
      "External id": 2480417,"Record function id": 0, "Ev Idx": 8120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471268.050, "dur": 10.165,
    "args": {
      "External id": 2480418,"Record function id": 0, "Concrete Inputs": ["[6423040]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471291.833, "dur": 34.236,
    "args": {
      "External id": 2480419,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471297.208, "dur": 2.643,
    "args": {
      "External id": 2480420,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471303.921, "dur": 0.480,
    "args": {
      "External id": 2480421,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "256"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471305.513, "dur": 0.913,
    "args": {
      "External id": 2480422,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "524544"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471307.595, "dur": 0.886,
    "args": {
      "External id": 2480423,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1048832"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471311.002, "dur": 0.655,
    "args": {
      "External id": 2480424,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "1573120"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471312.754, "dur": 0.507,
    "args": {
      "External id": 2480425,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "2097408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471314.405, "dur": 2.080,
    "args": {
      "External id": 2480426,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "2097664"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471317.722, "dur": 0.861,
    "args": {
      "External id": 2480427,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "3539456"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471319.668, "dur": 0.773,
    "args": {
      "External id": 2480428,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "4981248"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471337.121, "dur": 42.122,
    "args": {
      "External id": 2480429,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::all_gather_copy_in", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471412.154, "dur": 107.906,
    "args": {
      "External id": 2480430,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "6423040", "8", "4", "15", ""], "Input type": ["TensorList", "ScalarList", "Scalar", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [], [], [], [], [], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [], [], [], [], [], []], "Ev Idx": 8133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471422.722, "dur": 4.248,
    "args": {
      "External id": 2480431,"Record function id": 0, "Concrete Inputs": ["[51384320]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::narrow", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471431.638, "dur": 10.224,
    "args": {
      "External id": 2480432,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "6423040"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471436.278, "dur": 5.172,
    "args": {
      "External id": 2480433,"Record function id": 0, "Concrete Inputs": ["", "0", "25692160", "32115200", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[51384320], [], [], [], []], "Ev Idx": 8136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471439.692, "dur": 0.594,
    "args": {
      "External id": 2480434,"Record function id": 0, "Concrete Inputs": ["", "[6423040]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[51384320], [], [], []], "Ev Idx": 8137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471448.309, "dur": 26.460,
    "args": {
      "External id": 2480435,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[6423040], [], []], "Ev Idx": 8138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471450.260, "dur": 2.252,
    "args": {
      "External id": 2480436,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "25692160"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471454.003, "dur": 0.756,
    "args": {
      "External id": 2480437,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "25692416"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471455.836, "dur": 0.751,
    "args": {
      "External id": 2480438,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26216704"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471459.420, "dur": 0.554,
    "args": {
      "External id": 2480439,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "26740992"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471460.760, "dur": 0.667,
    "args": {
      "External id": 2480440,"Record function id": 0, "Concrete Inputs": ["", "[524288]", "[1]", "27265280"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471462.404, "dur": 0.442,
    "args": {
      "External id": 2480441,"Record function id": 0, "Concrete Inputs": ["", "[256]", "[1]", "27789568"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471465.405, "dur": 0.834,
    "args": {
      "External id": 2480442,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "27789824"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471467.120, "dur": 0.797,
    "args": {
      "External id": 2480443,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "29231616"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471468.858, "dur": 1.842,
    "args": {
      "External id": 2480444,"Record function id": 0, "Concrete Inputs": ["", "[1441792]", "[1]", "30673408"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[6423040], [], [], []], "Ev Idx": 8147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471487.218, "dur": 25.126,
    "args": {
      "External id": 2480445,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["TensorList", "TensorList", "Scalar"], "Input Strides": [[[1], [1], [1], [1], [1], [1], [1], [1], [1]], [[1], [1], [1], [1], [1], [1], [1], [1], [1]], []], "Input Dims": [[[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], [[256], [524288], [524288], [524288], [524288], [256], [1441792], [1441792], [1441792]], []], "Ev Idx": 8148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::_allgather_base_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471592.917, "dur": 368.657,
    "args": {
      "External id": 2480446,"Record function id": 0, "Concrete Inputs": ["", "", "", "False", "-1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "Scalar", "Scalar"], "Input Strides": [[1], [1], [], [], []], "Input Dims": [[51384320], [6423040], [], [], []], "Ev Idx": 8149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471624.147, "dur": 332.649,
    "args": {
      "External id": 2480447,"Record function id": 0, "Collective name": "_allgather_base", "Process Group Description": "default_pg", "dtype": "BFloat16", "Rank": 4, "Input Strides": [[1], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 51384320, "Process Group Name": "0", "Input type": ["c10::BFloat16", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[6423040], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 8150, "In msg nelems": 6423040
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:_all_gather_base", "pid": 1336757, "tid": 1336757,
    "ts": 1514656471634.965, "dur": 316.057,
    "args": {
      "External id": 2480448,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[1]], "Input Dims": [[6423040]], "Ev Idx": 8151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472017.396, "dur": 3.638,
    "args": {
      "External id": 2480449,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 8152, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::all_gather_copy_out (model.layers.27)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472117.150, "dur": 17106.168,
    "args": {
      "External id": 2480450,"Record function id": 0, "Ev Idx": 8153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472231.696, "dur": 6.918,
    "args": {
      "External id": 2480451,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[51384320], []], "Ev Idx": 8154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472241.971, "dur": 1.155,
    "args": {
      "External id": 2480452,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472244.933, "dur": 1.035,
    "args": {
      "External id": 2480453,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472249.053, "dur": 1.045,
    "args": {
      "External id": 2480454,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472251.358, "dur": 0.906,
    "args": {
      "External id": 2480455,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472253.571, "dur": 1.283,
    "args": {
      "External id": 2480456,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[4194304], []], "Ev Idx": 8159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472256.597, "dur": 1.117,
    "args": {
      "External id": 2480457,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[2048], []], "Ev Idx": 8160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472261.154, "dur": 2.110,
    "args": {
      "External id": 2480458,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472264.501, "dur": 1.121,
    "args": {
      "External id": 2480459,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472267.277, "dur": 0.940,
    "args": {
      "External id": 2480460,"Record function id": 0, "Concrete Inputs": ["", "[8, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[11534336], []], "Ev Idx": 8163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "fsdp::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472286.958, "dur": 16896.552,
    "args": {
      "External id": 2480461,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::split_with_sizes_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472301.986, "dur": 16874.873,
    "args": {
      "External id": 2480462,"Record function id": 0, "Concrete Inputs": ["", "[256, 524288, 524288, 524288, 524288, 256, 1441792, 1441792, 1441792]", "1", ""], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "TensorList"], "Input Strides": [[6423040, 1], [], [], [[256, 1], [524288, 1], [524288, 1], [524288, 1], [524288, 1], [256, 1], [1441792, 1], [1441792, 1], [1441792, 1]]], "Input Dims": [[8, 6423040], [], [], [[8, 256], [8, 524288], [8, 524288], [8, 524288], [8, 524288], [8, 256], [8, 1441792], [8, 1441792], [8, 1441792]]], "Ev Idx": 8165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472320.366, "dur": 14.318,
    "args": {
      "External id": 2480463,"Record function id": 0, "Concrete Inputs": ["[2133]", "4", "", "", "True", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472339.436, "dur": 16806.803,
    "args": {
      "External id": 2480464,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", "False", ""], "Input type": ["long int", "", "", "", "", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], [], []], "Ev Idx": 8167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472342.117, "dur": 16803.562,
    "args": {
      "External id": 2480465,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "True", ""], "Input type": ["long int", "", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[2133], [], [], [], [], [], []], "Ev Idx": 8168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472347.646, "dur": 5.473,
    "args": {
      "External id": 2480466,"Record function id": 0, "Concrete Inputs": ["[2133]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656472354.825, "dur": 16787.897,
    "args": {
      "External id": 2480467,"Record function id": 0, "Concrete Inputs": ["", "", "True"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[2133], [2133], []], "Ev Idx": 8170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RegisterPostBackwardFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656489357.944, "dur": 37.080,
    "args": {
      "External id": 2480468,"Sequence number": 24740475, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048]], "Ev Idx": 8171
    }
  },
  {
    "ph": "s", "id": 173, "pid": 1336757, "tid": 1336757, "ts": 1514656489357.944,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656489381.937, "dur": 8.157,
    "args": {
      "External id": 2480469,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1]], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048]], "Ev Idx": 8172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656489385.204, "dur": 4.655,
    "args": {
      "External id": 2480470,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656489456.154, "dur": 87.139,
    "args": {
      "External id": 2480471,"Record function id": 0, "Ev Idx": 8174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 1/1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656489546.066, "dur": 1052.754,
    "args": {
      "External id": 2480472,"Record function id": 0, "Ev Idx": 8175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656489586.698, "dur": 998.253,
    "args": {
      "External id": 2480473,"Sequence number": 24740476, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "", "", "", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1], [2048, 1], [2048, 1], [2048, 1], [32, 1], [32, 1], [2048, 1], [1], [2048, 1], [2048, 1], [5632, 1]], "Input Dims": [[16, 4096, 2048], [2048], [2048, 2048], [2048, 2048], [2048, 2048], [8192, 32], [8192, 32], [2048, 2048], [2048], [5632, 2048], [5632, 2048], [2048, 5632]], "Ev Idx": 8176
    }
  },
  {
    "ph": "s", "id": 172, "pid": 1336757, "tid": 1336757, "ts": 1514656489586.698,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656489653.406, "dur": 42.653,
    "args": {
      "External id": 2480474,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656489709.515, "dur": 104.913,
    "args": {
      "External id": 2480475,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656489823.694, "dur": 37.731,
    "args": {
      "External id": 2480476,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656489870.474, "dur": 30.977,
    "args": {
      "External id": 2480477,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656489923.958, "dur": 25.513,
    "args": {
      "External id": 2480478,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "rotary_embedding_kernel_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656489967.626, "dur": 48.317,
    "args": {
      "External id": 2480479,"kernel_hash": "ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "0", "16", "4096", "32", "64", "32", "8192", "32", "64", "False", "False", "False", "False"], "kernel_file": "/tmp/torchinductor_cvm/td/ctde6obw47lrxarntg442rinsuwlx4v7ftjlbooqhkhlhwhaksb4.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 8182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490035.516, "dur": 129.099,
    "args": {
      "External id": 2480480,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490085.897, "dur": 12.642,
    "args": {
      "External id": 2480481,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490090.837, "dur": 6.695,
    "args": {
      "External id": 2480482,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490101.287, "dur": 4.752,
    "args": {
      "External id": 2480483,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490107.513, "dur": 1.607,
    "args": {
      "External id": 2480484,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490111.357, "dur": 3.304,
    "args": {
      "External id": 2480485,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490176.174, "dur": 49.246,
    "args": {
      "External id": 2480486,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [2048, 1]], "Input Dims": [[65536, 2048], [2048, 2048], [65536, 2048]], "Ev Idx": 8189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_2", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490257.000, "dur": 28.016,
    "args": {
      "External id": 2480487,"kernel_hash": "cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "True", "True", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/oh/cohoteac6vn6qlofk3ytbewriosbrre4g473qjud7syi7ayj7ymc.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [2048, 1], [2048, 1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536, 2048], [2048], [65536, 2048], [65536, 2048], [65536], [], [], [], [], [], [], [], [], []], "Ev Idx": 8190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490293.217, "dur": 39.492,
    "args": {
      "External id": 2480488,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490341.316, "dur": 34.390,
    "args": {
      "External id": 2480489,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048], [5632, 1]], "Input Dims": [[65536, 2048], [2048, 5632], [65536, 5632]], "Ev Idx": 8192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused__to_copy_mul_silu_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490396.582, "dur": 28.410,
    "args": {
      "External id": 2480490,"kernel_hash": "cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf", "grid": "grid(369098752,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "369098752"], "kernel_file": "/tmp/torchinductor_cvm/qh/cqhqx2lwkkgnxczi2z4ew2shsidevhbk5pjz54fm6sbd6uftsmjf.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[5632, 1], [5632, 1], [23068672, 5632, 1], []], "Input Dims": [[65536, 5632], [65536, 5632], [16, 4096, 5632], []], "Ev Idx": 8193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490431.181, "dur": 32.819,
    "args": {
      "External id": 2480491,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632], [2048, 1]], "Input Dims": [[65536, 5632], [5632, 2048], [65536, 2048]], "Ev Idx": 8194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "triton_poi_fused_add_1", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490482.803, "dur": 17.223,
    "args": {
      "External id": 2480492,"kernel_hash": "cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu", "grid": "grid(134217728,)", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "134217728"], "kernel_file": "/tmp/torchinductor_cvm/qo/cqo6qtnieavgrnue4llkfxac3bydsk5p7r22potmkpxvtg3jxrsu.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [65536, 2048], []], "Ev Idx": 8195
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward (model.layers.27)", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490662.391, "dur": 31.114,
    "args": {
      "External id": 2480493,"Record function id": 0, "Ev Idx": 8196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490826.612, "dur": 332.063,
    "args": {
      "External id": 2480494,"Sequence number": 24740477, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8197
    }
  },
  {
    "ph": "s", "id": 171, "pid": 1336757, "tid": 1336757, "ts": 1514656490826.612,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490861.362, "dur": 8.561,
    "args": {
      "External id": 2480495,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490863.805, "dur": 5.850,
    "args": {
      "External id": 2480496,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490879.819, "dur": 12.104,
    "args": {
      "External id": 2480497,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490882.834, "dur": 8.395,
    "args": {
      "External id": 2480498,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656490901.666, "dur": 5.207,
    "args": {
      "External id": 2480499,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491135.124, "dur": 9.070,
    "args": {
      "External id": 2480500,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491139.281, "dur": 4.460,
    "args": {
      "External id": 2480501,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491188.941, "dur": 136.864,
    "args": {
      "External id": 2480502,"Sequence number": 24740478, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491192.784, "dur": 16.908,
    "args": {
      "External id": 2480503,"Sequence number": 24740478, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8206
    }
  },
  {
    "ph": "s", "id": 170, "pid": 1336757, "tid": 1336757, "ts": 1514656491192.784,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491199.374, "dur": 8.589,
    "args": {
      "External id": 2480504,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491205.490, "dur": 2.129,
    "args": {
      "External id": 2480505,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491212.357, "dur": 113.061,
    "args": {
      "External id": 2480506,"Sequence number": 24740479, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491215.916, "dur": 5.261,
    "args": {
      "External id": 2480507,"Sequence number": 24740479, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491216.597, "dur": 4.443,
    "args": {
      "External id": 2480508,"Sequence number": 24740479, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8211
    }
  },
  {
    "ph": "s", "id": 169, "pid": 1336757, "tid": 1336757, "ts": 1514656491216.597,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491225.219, "dur": 89.612,
    "args": {
      "External id": 2480509,"Sequence number": 24740480, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8212
    }
  },
  {
    "ph": "s", "id": 168, "pid": 1336757, "tid": 1336757, "ts": 1514656491225.219,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491318.255, "dur": 5.918,
    "args": {
      "External id": 2480510,"Sequence number": 24740481, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8213
    }
  },
  {
    "ph": "s", "id": 167, "pid": 1336757, "tid": 1336757, "ts": 1514656491318.255,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491335.205, "dur": 65.515,
    "args": {
      "External id": 2480511,"Sequence number": 24740482, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491336.068, "dur": 8.057,
    "args": {
      "External id": 2480512,"Sequence number": 24740482, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8215
    }
  },
  {
    "ph": "s", "id": 166, "pid": 1336757, "tid": 1336757, "ts": 1514656491336.068,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491337.953, "dur": 4.962,
    "args": {
      "External id": 2480513,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491341.526, "dur": 1.199,
    "args": {
      "External id": 2480514,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491344.803, "dur": 55.624,
    "args": {
      "External id": 2480515,"Sequence number": 24740483, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491345.791, "dur": 5.666,
    "args": {
      "External id": 2480516,"Sequence number": 24740483, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491346.700, "dur": 4.593,
    "args": {
      "External id": 2480517,"Sequence number": 24740483, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8220
    }
  },
  {
    "ph": "s", "id": 165, "pid": 1336757, "tid": 1336757, "ts": 1514656491346.700,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491352.175, "dur": 41.320,
    "args": {
      "External id": 2480518,"Sequence number": 24740484, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8221
    }
  },
  {
    "ph": "s", "id": 164, "pid": 1336757, "tid": 1336757, "ts": 1514656491352.175,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491395.273, "dur": 4.664,
    "args": {
      "External id": 2480519,"Sequence number": 24740485, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8222
    }
  },
  {
    "ph": "s", "id": 163, "pid": 1336757, "tid": 1336757, "ts": 1514656491395.273,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491408.986, "dur": 62.419,
    "args": {
      "External id": 2480520,"Sequence number": 24740486, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491409.677, "dur": 6.047,
    "args": {
      "External id": 2480521,"Sequence number": 24740486, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8224
    }
  },
  {
    "ph": "s", "id": 162, "pid": 1336757, "tid": 1336757, "ts": 1514656491409.677,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491411.928, "dur": 2.722,
    "args": {
      "External id": 2480522,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491413.361, "dur": 0.984,
    "args": {
      "External id": 2480523,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491417.908, "dur": 53.235,
    "args": {
      "External id": 2480524,"Sequence number": 24740487, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491419.085, "dur": 4.673,
    "args": {
      "External id": 2480525,"Sequence number": 24740487, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491419.910, "dur": 3.704,
    "args": {
      "External id": 2480526,"Sequence number": 24740487, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8229
    }
  },
  {
    "ph": "s", "id": 161, "pid": 1336757, "tid": 1336757, "ts": 1514656491419.910,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491424.294, "dur": 39.507,
    "args": {
      "External id": 2480527,"Sequence number": 24740488, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8230
    }
  },
  {
    "ph": "s", "id": 160, "pid": 1336757, "tid": 1336757, "ts": 1514656491424.294,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491465.588, "dur": 5.112,
    "args": {
      "External id": 2480528,"Sequence number": 24740489, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8231
    }
  },
  {
    "ph": "s", "id": 159, "pid": 1336757, "tid": 1336757, "ts": 1514656491465.588,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491493.333, "dur": 3.706,
    "args": {
      "External id": 2480529,"Sequence number": 24740490, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491494.244, "dur": 2.612,
    "args": {
      "External id": 2480530,"Sequence number": 24740490, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8233
    }
  },
  {
    "ph": "s", "id": 158, "pid": 1336757, "tid": 1336757, "ts": 1514656491494.244,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491503.737, "dur": 3.246,
    "args": {
      "External id": 2480531,"Sequence number": 24740491, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491504.733, "dur": 2.082,
    "args": {
      "External id": 2480532,"Sequence number": 24740491, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8235
    }
  },
  {
    "ph": "s", "id": 157, "pid": 1336757, "tid": 1336757, "ts": 1514656491504.733,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491512.361, "dur": 4.254,
    "args": {
      "External id": 2480533,"Sequence number": 24740492, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491513.294, "dur": 3.184,
    "args": {
      "External id": 2480534,"Sequence number": 24740492, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8237
    }
  },
  {
    "ph": "s", "id": 156, "pid": 1336757, "tid": 1336757, "ts": 1514656491513.294,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491569.860, "dur": 186.768,
    "args": {
      "External id": 2480535,"Sequence number": 24740493, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8238
    }
  },
  {
    "ph": "s", "id": 155, "pid": 1336757, "tid": 1336757, "ts": 1514656491569.860,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491593.692, "dur": 9.993,
    "args": {
      "External id": 2480536,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491596.953, "dur": 6.066,
    "args": {
      "External id": 2480537,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491769.295, "dur": 111.315,
    "args": {
      "External id": 2480538,"Sequence number": 24740494, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8241
    }
  },
  {
    "ph": "s", "id": 154, "pid": 1336757, "tid": 1336757, "ts": 1514656491769.295,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491783.462, "dur": 7.609,
    "args": {
      "External id": 2480539,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491786.184, "dur": 4.465,
    "args": {
      "External id": 2480540,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491911.943, "dur": 233.247,
    "args": {
      "External id": 2480541,"Sequence number": 24740495, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8244
    }
  },
  {
    "ph": "s", "id": 153, "pid": 1336757, "tid": 1336757, "ts": 1514656491911.943,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656491942.510, "dur": 168.369,
    "args": {
      "External id": 2480542,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492028.084, "dur": 8.270,
    "args": {
      "External id": 2480543,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492030.490, "dur": 5.158,
    "args": {
      "External id": 2480544,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492039.113, "dur": 4.602,
    "args": {
      "External id": 2480545,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492045.423, "dur": 1.361,
    "args": {
      "External id": 2480546,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492050.775, "dur": 3.407,
    "args": {
      "External id": 2480547,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492126.225, "dur": 8.061,
    "args": {
      "External id": 2480548,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492150.105, "dur": 6.374,
    "args": {
      "External id": 2480549,"Sequence number": 24740496, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492151.748, "dur": 4.540,
    "args": {
      "External id": 2480550,"Sequence number": 24740496, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8253
    }
  },
  {
    "ph": "s", "id": 152, "pid": 1336757, "tid": 1336757, "ts": 1514656492151.748,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492173.145, "dur": 120.944,
    "args": {
      "External id": 2480551,"Sequence number": 24740497, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492176.257, "dur": 8.470,
    "args": {
      "External id": 2480552,"Sequence number": 24740497, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8255
    }
  },
  {
    "ph": "s", "id": 151, "pid": 1336757, "tid": 1336757, "ts": 1514656492176.257,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492178.985, "dur": 4.496,
    "args": {
      "External id": 2480553,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492181.220, "dur": 1.983,
    "args": {
      "External id": 2480554,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492185.903, "dur": 107.881,
    "args": {
      "External id": 2480555,"Sequence number": 24740498, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492187.661, "dur": 8.285,
    "args": {
      "External id": 2480556,"Sequence number": 24740498, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492190.013, "dur": 5.767,
    "args": {
      "External id": 2480557,"Sequence number": 24740498, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8260
    }
  },
  {
    "ph": "s", "id": 150, "pid": 1336757, "tid": 1336757, "ts": 1514656492190.013,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492196.974, "dur": 88.718,
    "args": {
      "External id": 2480558,"Sequence number": 24740499, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8261
    }
  },
  {
    "ph": "s", "id": 149, "pid": 1336757, "tid": 1336757, "ts": 1514656492196.974,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492288.078, "dur": 4.920,
    "args": {
      "External id": 2480559,"Sequence number": 24740500, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8262
    }
  },
  {
    "ph": "s", "id": 148, "pid": 1336757, "tid": 1336757, "ts": 1514656492288.078,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492330.809, "dur": 246.378,
    "args": {
      "External id": 2480560,"Sequence number": 24740501, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8263
    }
  },
  {
    "ph": "s", "id": 147, "pid": 1336757, "tid": 1336757, "ts": 1514656492330.809,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492352.238, "dur": 3.202,
    "args": {
      "External id": 2480561,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492353.057, "dur": 2.023,
    "args": {
      "External id": 2480562,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492361.881, "dur": 3.773,
    "args": {
      "External id": 2480563,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492363.478, "dur": 2.072,
    "args": {
      "External id": 2480564,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492364.308, "dur": 1.123,
    "args": {
      "External id": 2480565,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492373.215, "dur": 8.098,
    "args": {
      "External id": 2480566,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492376.924, "dur": 3.989,
    "args": {
      "External id": 2480567,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492387.720, "dur": 3.346,
    "args": {
      "External id": 2480568,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492394.638, "dur": 2.962,
    "args": {
      "External id": 2480569,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492550.597, "dur": 4.211,
    "args": {
      "External id": 2480570,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492551.584, "dur": 2.806,
    "args": {
      "External id": 2480571,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492557.636, "dur": 2.121,
    "args": {
      "External id": 2480572,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492558.371, "dur": 1.282,
    "args": {
      "External id": 2480573,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492598.392, "dur": 94.574,
    "args": {
      "External id": 2480574,"Sequence number": 24740502, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492599.449, "dur": 7.913,
    "args": {
      "External id": 2480575,"Sequence number": 24740502, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8278
    }
  },
  {
    "ph": "s", "id": 146, "pid": 1336757, "tid": 1336757, "ts": 1514656492599.449,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492601.938, "dur": 4.191,
    "args": {
      "External id": 2480576,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492604.145, "dur": 1.688,
    "args": {
      "External id": 2480577,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492608.346, "dur": 84.351,
    "args": {
      "External id": 2480578,"Sequence number": 24740503, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492611.589, "dur": 3.589,
    "args": {
      "External id": 2480579,"Sequence number": 24740503, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492612.422, "dur": 2.593,
    "args": {
      "External id": 2480580,"Sequence number": 24740503, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8283
    }
  },
  {
    "ph": "s", "id": 145, "pid": 1336757, "tid": 1336757, "ts": 1514656492612.422,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492615.915, "dur": 71.317,
    "args": {
      "External id": 2480581,"Sequence number": 24740504, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8284
    }
  },
  {
    "ph": "s", "id": 144, "pid": 1336757, "tid": 1336757, "ts": 1514656492615.915,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492689.260, "dur": 2.757,
    "args": {
      "External id": 2480582,"Sequence number": 24740505, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8285
    }
  },
  {
    "ph": "s", "id": 143, "pid": 1336757, "tid": 1336757, "ts": 1514656492689.260,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492701.758, "dur": 67.333,
    "args": {
      "External id": 2480583,"Sequence number": 24740506, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492703.864, "dur": 6.285,
    "args": {
      "External id": 2480584,"Sequence number": 24740506, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8287
    }
  },
  {
    "ph": "s", "id": 142, "pid": 1336757, "tid": 1336757, "ts": 1514656492703.864,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492706.170, "dur": 2.768,
    "args": {
      "External id": 2480585,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492707.916, "dur": 0.875,
    "args": {
      "External id": 2480586,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492710.800, "dur": 58.045,
    "args": {
      "External id": 2480587,"Sequence number": 24740507, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492711.671, "dur": 7.062,
    "args": {
      "External id": 2480588,"Sequence number": 24740507, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492713.853, "dur": 4.729,
    "args": {
      "External id": 2480589,"Sequence number": 24740507, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8292
    }
  },
  {
    "ph": "s", "id": 141, "pid": 1336757, "tid": 1336757, "ts": 1514656492713.853,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492719.281, "dur": 43.274,
    "args": {
      "External id": 2480590,"Sequence number": 24740508, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8293
    }
  },
  {
    "ph": "s", "id": 140, "pid": 1336757, "tid": 1336757, "ts": 1514656492719.281,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492764.362, "dur": 4.029,
    "args": {
      "External id": 2480591,"Sequence number": 24740509, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8294
    }
  },
  {
    "ph": "s", "id": 139, "pid": 1336757, "tid": 1336757, "ts": 1514656492764.362,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492791.415, "dur": 160.065,
    "args": {
      "External id": 2480592,"Sequence number": 24740510, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8295
    }
  },
  {
    "ph": "s", "id": 138, "pid": 1336757, "tid": 1336757, "ts": 1514656492791.415,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492840.217, "dur": 3.844,
    "args": {
      "External id": 2480593,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492881.609, "dur": 57.078,
    "args": {
      "External id": 2480594,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492882.195, "dur": 5.087,
    "args": {
      "External id": 2480595,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492883.569, "dur": 3.032,
    "args": {
      "External id": 2480596,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492885.091, "dur": 1.320,
    "args": {
      "External id": 2480597,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492887.976, "dur": 50.289,
    "args": {
      "External id": 2480598,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492891.161, "dur": 2.469,
    "args": {
      "External id": 2480599,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492891.928, "dur": 1.541,
    "args": {
      "External id": 2480600,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492894.188, "dur": 40.042,
    "args": {
      "External id": 2480601,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492936.380, "dur": 1.225,
    "args": {
      "External id": 2480602,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1336757,
    "ts": 1514656492961.771, "dur": 65.346,
    "args": {
      "External id": 2480603,"Sequence number": 24740511, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8306
    }
  },
  {
    "ph": "s", "id": 137, "pid": 1336757, "tid": 1336757, "ts": 1514656492961.771,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493066.786, "dur": 187.648,
    "args": {
      "External id": 2480604,"Sequence number": 24740512, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8307
    }
  },
  {
    "ph": "s", "id": 136, "pid": 1336757, "tid": 1336757, "ts": 1514656493066.786,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493089.292, "dur": 5.410,
    "args": {
      "External id": 2480605,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493091.977, "dur": 2.545,
    "args": {
      "External id": 2480606,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493102.701, "dur": 7.878,
    "args": {
      "External id": 2480607,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493105.512, "dur": 4.566,
    "args": {
      "External id": 2480608,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493116.620, "dur": 3.591,
    "args": {
      "External id": 2480609,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493239.268, "dur": 3.166,
    "args": {
      "External id": 2480610,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493240.454, "dur": 1.731,
    "args": {
      "External id": 2480611,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493273.188, "dur": 85.134,
    "args": {
      "External id": 2480612,"Sequence number": 24740513, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493274.308, "dur": 8.674,
    "args": {
      "External id": 2480613,"Sequence number": 24740513, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8316
    }
  },
  {
    "ph": "s", "id": 135, "pid": 1336757, "tid": 1336757, "ts": 1514656493274.308,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493277.949, "dur": 3.654,
    "args": {
      "External id": 2480614,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493280.008, "dur": 1.363,
    "args": {
      "External id": 2480615,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493283.991, "dur": 74.029,
    "args": {
      "External id": 2480616,"Sequence number": 24740514, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493285.361, "dur": 4.935,
    "args": {
      "External id": 2480617,"Sequence number": 24740514, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493287.602, "dur": 2.576,
    "args": {
      "External id": 2480618,"Sequence number": 24740514, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8321
    }
  },
  {
    "ph": "s", "id": 134, "pid": 1336757, "tid": 1336757, "ts": 1514656493287.602,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493291.229, "dur": 61.251,
    "args": {
      "External id": 2480619,"Sequence number": 24740515, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8322
    }
  },
  {
    "ph": "s", "id": 133, "pid": 1336757, "tid": 1336757, "ts": 1514656493291.229,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493354.700, "dur": 2.605,
    "args": {
      "External id": 2480620,"Sequence number": 24740516, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8323
    }
  },
  {
    "ph": "s", "id": 132, "pid": 1336757, "tid": 1336757, "ts": 1514656493354.700,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493366.707, "dur": 63.077,
    "args": {
      "External id": 2480621,"Sequence number": 24740517, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493367.510, "dur": 8.208,
    "args": {
      "External id": 2480622,"Sequence number": 24740517, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8325
    }
  },
  {
    "ph": "s", "id": 131, "pid": 1336757, "tid": 1336757, "ts": 1514656493367.510,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493372.209, "dur": 2.521,
    "args": {
      "External id": 2480623,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493373.819, "dur": 0.752,
    "args": {
      "External id": 2480624,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493376.393, "dur": 53.082,
    "args": {
      "External id": 2480625,"Sequence number": 24740518, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493377.647, "dur": 4.424,
    "args": {
      "External id": 2480626,"Sequence number": 24740518, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493378.253, "dur": 3.710,
    "args": {
      "External id": 2480627,"Sequence number": 24740518, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8330
    }
  },
  {
    "ph": "s", "id": 130, "pid": 1336757, "tid": 1336757, "ts": 1514656493378.253,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493382.618, "dur": 42.276,
    "args": {
      "External id": 2480628,"Sequence number": 24740519, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8331
    }
  },
  {
    "ph": "s", "id": 129, "pid": 1336757, "tid": 1336757, "ts": 1514656493382.618,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493426.742, "dur": 2.353,
    "args": {
      "External id": 2480629,"Sequence number": 24740520, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8332
    }
  },
  {
    "ph": "s", "id": 128, "pid": 1336757, "tid": 1336757, "ts": 1514656493426.742,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493436.488, "dur": 58.724,
    "args": {
      "External id": 2480630,"Sequence number": 24740521, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493437.154, "dur": 8.548,
    "args": {
      "External id": 2480631,"Sequence number": 24740521, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8334
    }
  },
  {
    "ph": "s", "id": 127, "pid": 1336757, "tid": 1336757, "ts": 1514656493437.154,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493438.936, "dur": 5.637,
    "args": {
      "External id": 2480632,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493442.159, "dur": 2.197,
    "args": {
      "External id": 2480633,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493446.233, "dur": 48.661,
    "args": {
      "External id": 2480634,"Sequence number": 24740522, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493446.879, "dur": 3.784,
    "args": {
      "External id": 2480635,"Sequence number": 24740522, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493447.840, "dur": 2.697,
    "args": {
      "External id": 2480636,"Sequence number": 24740522, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8339
    }
  },
  {
    "ph": "s", "id": 126, "pid": 1336757, "tid": 1336757, "ts": 1514656493447.840,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493452.885, "dur": 36.454,
    "args": {
      "External id": 2480637,"Sequence number": 24740523, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8340
    }
  },
  {
    "ph": "s", "id": 125, "pid": 1336757, "tid": 1336757, "ts": 1514656493452.885,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493490.892, "dur": 3.669,
    "args": {
      "External id": 2480638,"Sequence number": 24740524, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8341
    }
  },
  {
    "ph": "s", "id": 124, "pid": 1336757, "tid": 1336757, "ts": 1514656493490.892,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493509.271, "dur": 3.787,
    "args": {
      "External id": 2480639,"Sequence number": 24740525, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493510.339, "dur": 2.545,
    "args": {
      "External id": 2480640,"Sequence number": 24740525, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8343
    }
  },
  {
    "ph": "s", "id": 123, "pid": 1336757, "tid": 1336757, "ts": 1514656493510.339,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493566.008, "dur": 8.840,
    "args": {
      "External id": 2480641,"Sequence number": 24740526, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493570.270, "dur": 4.332,
    "args": {
      "External id": 2480642,"Sequence number": 24740526, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8345
    }
  },
  {
    "ph": "s", "id": 122, "pid": 1336757, "tid": 1336757, "ts": 1514656493570.270,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493581.196, "dur": 4.511,
    "args": {
      "External id": 2480643,"Sequence number": 24740527, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493582.158, "dur": 3.376,
    "args": {
      "External id": 2480644,"Sequence number": 24740527, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8347
    }
  },
  {
    "ph": "s", "id": 121, "pid": 1336757, "tid": 1336757, "ts": 1514656493582.158,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493614.621, "dur": 151.692,
    "args": {
      "External id": 2480645,"Sequence number": 24740528, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8348
    }
  },
  {
    "ph": "s", "id": 120, "pid": 1336757, "tid": 1336757, "ts": 1514656493614.621,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493633.925, "dur": 8.605,
    "args": {
      "External id": 2480646,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493636.308, "dur": 5.630,
    "args": {
      "External id": 2480647,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493779.816, "dur": 98.972,
    "args": {
      "External id": 2480648,"Sequence number": 24740529, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8351
    }
  },
  {
    "ph": "s", "id": 119, "pid": 1336757, "tid": 1336757, "ts": 1514656493779.816,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493793.202, "dur": 6.156,
    "args": {
      "External id": 2480649,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493795.029, "dur": 4.027,
    "args": {
      "External id": 2480650,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493905.097, "dur": 215.283,
    "args": {
      "External id": 2480651,"Sequence number": 24740530, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8354
    }
  },
  {
    "ph": "s", "id": 118, "pid": 1336757, "tid": 1336757, "ts": 1514656493905.097,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493931.684, "dur": 158.972,
    "args": {
      "External id": 2480652,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493974.949, "dur": 7.401,
    "args": {
      "External id": 2480653,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656493977.117, "dur": 4.725,
    "args": {
      "External id": 2480654,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494020.446, "dur": 5.347,
    "args": {
      "External id": 2480655,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494027.472, "dur": 2.797,
    "args": {
      "External id": 2480656,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494033.007, "dur": 3.109,
    "args": {
      "External id": 2480657,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494103.289, "dur": 5.597,
    "args": {
      "External id": 2480658,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494125.568, "dur": 7.427,
    "args": {
      "External id": 2480659,"Sequence number": 24740531, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494126.748, "dur": 6.100,
    "args": {
      "External id": 2480660,"Sequence number": 24740531, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8363
    }
  },
  {
    "ph": "s", "id": 117, "pid": 1336757, "tid": 1336757, "ts": 1514656494126.748,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494148.898, "dur": 114.470,
    "args": {
      "External id": 2480661,"Sequence number": 24740532, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494150.248, "dur": 13.560,
    "args": {
      "External id": 2480662,"Sequence number": 24740532, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8365
    }
  },
  {
    "ph": "s", "id": 116, "pid": 1336757, "tid": 1336757, "ts": 1514656494150.248,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494152.613, "dur": 10.043,
    "args": {
      "External id": 2480663,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494160.582, "dur": 1.787,
    "args": {
      "External id": 2480664,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494165.048, "dur": 97.958,
    "args": {
      "External id": 2480665,"Sequence number": 24740533, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494168.312, "dur": 4.279,
    "args": {
      "External id": 2480666,"Sequence number": 24740533, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494168.812, "dur": 3.574,
    "args": {
      "External id": 2480667,"Sequence number": 24740533, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8370
    }
  },
  {
    "ph": "s", "id": 115, "pid": 1336757, "tid": 1336757, "ts": 1514656494168.812,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494173.300, "dur": 81.845,
    "args": {
      "External id": 2480668,"Sequence number": 24740534, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8371
    }
  },
  {
    "ph": "s", "id": 114, "pid": 1336757, "tid": 1336757, "ts": 1514656494173.300,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494257.378, "dur": 4.812,
    "args": {
      "External id": 2480669,"Sequence number": 24740535, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8372
    }
  },
  {
    "ph": "s", "id": 113, "pid": 1336757, "tid": 1336757, "ts": 1514656494257.378,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494298.023, "dur": 210.652,
    "args": {
      "External id": 2480670,"Sequence number": 24740536, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8373
    }
  },
  {
    "ph": "s", "id": 112, "pid": 1336757, "tid": 1336757, "ts": 1514656494298.023,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494320.498, "dur": 2.298,
    "args": {
      "External id": 2480671,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494321.168, "dur": 1.486,
    "args": {
      "External id": 2480672,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494326.414, "dur": 4.403,
    "args": {
      "External id": 2480673,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494327.134, "dur": 3.573,
    "args": {
      "External id": 2480674,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494329.727, "dur": 0.791,
    "args": {
      "External id": 2480675,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494338.152, "dur": 6.401,
    "args": {
      "External id": 2480676,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494339.856, "dur": 4.379,
    "args": {
      "External id": 2480677,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494350.674, "dur": 2.978,
    "args": {
      "External id": 2480678,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494357.351, "dur": 2.749,
    "args": {
      "External id": 2480679,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494486.343, "dur": 3.041,
    "args": {
      "External id": 2480680,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494487.357, "dur": 1.895,
    "args": {
      "External id": 2480681,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494493.330, "dur": 2.089,
    "args": {
      "External id": 2480682,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494494.186, "dur": 1.095,
    "args": {
      "External id": 2480683,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494525.840, "dur": 115.033,
    "args": {
      "External id": 2480684,"Sequence number": 24740537, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494544.222, "dur": 8.658,
    "args": {
      "External id": 2480685,"Sequence number": 24740537, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8388
    }
  },
  {
    "ph": "s", "id": 111, "pid": 1336757, "tid": 1336757, "ts": 1514656494544.222,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494546.949, "dur": 4.338,
    "args": {
      "External id": 2480686,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494549.108, "dur": 1.746,
    "args": {
      "External id": 2480687,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494555.563, "dur": 84.954,
    "args": {
      "External id": 2480688,"Sequence number": 24740538, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494556.806, "dur": 3.791,
    "args": {
      "External id": 2480689,"Sequence number": 24740538, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494557.608, "dur": 2.865,
    "args": {
      "External id": 2480690,"Sequence number": 24740538, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8393
    }
  },
  {
    "ph": "s", "id": 110, "pid": 1336757, "tid": 1336757, "ts": 1514656494557.608,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494561.264, "dur": 72.494,
    "args": {
      "External id": 2480691,"Sequence number": 24740539, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8394
    }
  },
  {
    "ph": "s", "id": 109, "pid": 1336757, "tid": 1336757, "ts": 1514656494561.264,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494635.834, "dur": 4.047,
    "args": {
      "External id": 2480692,"Sequence number": 24740540, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8395
    }
  },
  {
    "ph": "s", "id": 108, "pid": 1336757, "tid": 1336757, "ts": 1514656494635.834,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494650.683, "dur": 64.209,
    "args": {
      "External id": 2480693,"Sequence number": 24740541, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494651.534, "dur": 4.772,
    "args": {
      "External id": 2480694,"Sequence number": 24740541, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8397
    }
  },
  {
    "ph": "s", "id": 107, "pid": 1336757, "tid": 1336757, "ts": 1514656494651.534,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494652.721, "dur": 2.551,
    "args": {
      "External id": 2480695,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494654.156, "dur": 0.927,
    "args": {
      "External id": 2480696,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494656.962, "dur": 57.710,
    "args": {
      "External id": 2480697,"Sequence number": 24740542, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494659.781, "dur": 5.672,
    "args": {
      "External id": 2480698,"Sequence number": 24740542, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494660.234, "dur": 5.060,
    "args": {
      "External id": 2480699,"Sequence number": 24740542, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8402
    }
  },
  {
    "ph": "s", "id": 106, "pid": 1336757, "tid": 1336757, "ts": 1514656494660.234,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494665.951, "dur": 43.425,
    "args": {
      "External id": 2480700,"Sequence number": 24740543, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8403
    }
  },
  {
    "ph": "s", "id": 105, "pid": 1336757, "tid": 1336757, "ts": 1514656494665.951,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494710.851, "dur": 3.431,
    "args": {
      "External id": 2480701,"Sequence number": 24740544, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8404
    }
  },
  {
    "ph": "s", "id": 104, "pid": 1336757, "tid": 1336757, "ts": 1514656494710.851,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494737.550, "dur": 138.228,
    "args": {
      "External id": 2480702,"Sequence number": 24740545, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8405
    }
  },
  {
    "ph": "s", "id": 103, "pid": 1336757, "tid": 1336757, "ts": 1514656494737.550,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494774.209, "dur": 3.878,
    "args": {
      "External id": 2480703,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494808.400, "dur": 54.592,
    "args": {
      "External id": 2480704,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494809.264, "dur": 4.298,
    "args": {
      "External id": 2480705,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494810.284, "dur": 2.491,
    "args": {
      "External id": 2480706,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494811.759, "dur": 0.835,
    "args": {
      "External id": 2480707,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494815.970, "dur": 46.770,
    "args": {
      "External id": 2480708,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494816.901, "dur": 2.188,
    "args": {
      "External id": 2480709,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494817.720, "dur": 1.235,
    "args": {
      "External id": 2480710,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494819.582, "dur": 39.900,
    "args": {
      "External id": 2480711,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494861.178, "dur": 0.899,
    "args": {
      "External id": 2480712,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494883.824, "dur": 22.398,
    "args": {
      "External id": 2480713,"Sequence number": 24740546, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8416
    }
  },
  {
    "ph": "s", "id": 102, "pid": 1336757, "tid": 1336757, "ts": 1514656494883.824,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494939.297, "dur": 224.332,
    "args": {
      "External id": 2480714,"Sequence number": 24740547, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8417
    }
  },
  {
    "ph": "s", "id": 101, "pid": 1336757, "tid": 1336757, "ts": 1514656494939.297,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494957.863, "dur": 3.074,
    "args": {
      "External id": 2480715,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494958.547, "dur": 2.180,
    "args": {
      "External id": 2480716,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494968.117, "dur": 6.667,
    "args": {
      "External id": 2480717,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494970.215, "dur": 4.069,
    "args": {
      "External id": 2480718,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656494980.201, "dur": 3.498,
    "args": {
      "External id": 2480719,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495143.998, "dur": 4.282,
    "args": {
      "External id": 2480720,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495145.039, "dur": 2.803,
    "args": {
      "External id": 2480721,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495182.075, "dur": 87.416,
    "args": {
      "External id": 2480722,"Sequence number": 24740548, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495183.083, "dur": 9.008,
    "args": {
      "External id": 2480723,"Sequence number": 24740548, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8426
    }
  },
  {
    "ph": "s", "id": 100, "pid": 1336757, "tid": 1336757, "ts": 1514656495183.083,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495185.684, "dur": 4.940,
    "args": {
      "External id": 2480724,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495189.160, "dur": 1.260,
    "args": {
      "External id": 2480725,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495193.028, "dur": 76.129,
    "args": {
      "External id": 2480726,"Sequence number": 24740549, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495194.718, "dur": 3.207,
    "args": {
      "External id": 2480727,"Sequence number": 24740549, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495195.172, "dur": 2.616,
    "args": {
      "External id": 2480728,"Sequence number": 24740549, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8431
    }
  },
  {
    "ph": "s", "id": 99, "pid": 1336757, "tid": 1336757, "ts": 1514656495195.172,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495198.579, "dur": 63.396,
    "args": {
      "External id": 2480729,"Sequence number": 24740550, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8432
    }
  },
  {
    "ph": "s", "id": 98, "pid": 1336757, "tid": 1336757, "ts": 1514656495198.579,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495264.119, "dur": 4.272,
    "args": {
      "External id": 2480730,"Sequence number": 24740551, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8433
    }
  },
  {
    "ph": "s", "id": 97, "pid": 1336757, "tid": 1336757, "ts": 1514656495264.119,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495277.592, "dur": 59.067,
    "args": {
      "External id": 2480731,"Sequence number": 24740552, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495278.282, "dur": 5.709,
    "args": {
      "External id": 2480732,"Sequence number": 24740552, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8435
    }
  },
  {
    "ph": "s", "id": 96, "pid": 1336757, "tid": 1336757, "ts": 1514656495278.282,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495279.913, "dur": 3.061,
    "args": {
      "External id": 2480733,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495281.676, "dur": 1.150,
    "args": {
      "External id": 2480734,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495286.253, "dur": 50.132,
    "args": {
      "External id": 2480735,"Sequence number": 24740553, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495287.249, "dur": 2.937,
    "args": {
      "External id": 2480736,"Sequence number": 24740553, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495288.059, "dur": 1.985,
    "args": {
      "External id": 2480737,"Sequence number": 24740553, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8440
    }
  },
  {
    "ph": "s", "id": 95, "pid": 1336757, "tid": 1336757, "ts": 1514656495288.059,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495290.666, "dur": 40.579,
    "args": {
      "External id": 2480738,"Sequence number": 24740554, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8441
    }
  },
  {
    "ph": "s", "id": 94, "pid": 1336757, "tid": 1336757, "ts": 1514656495290.666,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495332.936, "dur": 3.080,
    "args": {
      "External id": 2480739,"Sequence number": 24740555, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8442
    }
  },
  {
    "ph": "s", "id": 93, "pid": 1336757, "tid": 1336757, "ts": 1514656495332.936,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495343.975, "dur": 57.357,
    "args": {
      "External id": 2480740,"Sequence number": 24740556, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495344.542, "dur": 6.598,
    "args": {
      "External id": 2480741,"Sequence number": 24740556, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8444
    }
  },
  {
    "ph": "s", "id": 92, "pid": 1336757, "tid": 1336757, "ts": 1514656495344.542,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495346.030, "dur": 4.078,
    "args": {
      "External id": 2480742,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495347.255, "dur": 2.621,
    "args": {
      "External id": 2480743,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495351.772, "dur": 49.335,
    "args": {
      "External id": 2480744,"Sequence number": 24740557, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495354.465, "dur": 3.834,
    "args": {
      "External id": 2480745,"Sequence number": 24740557, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495354.918, "dur": 3.258,
    "args": {
      "External id": 2480746,"Sequence number": 24740557, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8449
    }
  },
  {
    "ph": "s", "id": 91, "pid": 1336757, "tid": 1336757, "ts": 1514656495354.918,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495358.753, "dur": 36.650,
    "args": {
      "External id": 2480747,"Sequence number": 24740558, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8450
    }
  },
  {
    "ph": "s", "id": 90, "pid": 1336757, "tid": 1336757, "ts": 1514656495358.753,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495396.820, "dur": 3.956,
    "args": {
      "External id": 2480748,"Sequence number": 24740559, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8451
    }
  },
  {
    "ph": "s", "id": 89, "pid": 1336757, "tid": 1336757, "ts": 1514656495396.820,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495417.438, "dur": 3.286,
    "args": {
      "External id": 2480749,"Sequence number": 24740560, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495417.998, "dur": 2.599,
    "args": {
      "External id": 2480750,"Sequence number": 24740560, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8453
    }
  },
  {
    "ph": "s", "id": 88, "pid": 1336757, "tid": 1336757, "ts": 1514656495417.998,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495427.275, "dur": 2.887,
    "args": {
      "External id": 2480751,"Sequence number": 24740561, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495428.173, "dur": 1.854,
    "args": {
      "External id": 2480752,"Sequence number": 24740561, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8455
    }
  },
  {
    "ph": "s", "id": 87, "pid": 1336757, "tid": 1336757, "ts": 1514656495428.173,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495434.217, "dur": 4.050,
    "args": {
      "External id": 2480753,"Sequence number": 24740562, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495436.433, "dur": 1.716,
    "args": {
      "External id": 2480754,"Sequence number": 24740562, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8457
    }
  },
  {
    "ph": "s", "id": 86, "pid": 1336757, "tid": 1336757, "ts": 1514656495436.433,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495464.663, "dur": 171.504,
    "args": {
      "External id": 2480755,"Sequence number": 24740563, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8458
    }
  },
  {
    "ph": "s", "id": 85, "pid": 1336757, "tid": 1336757, "ts": 1514656495464.663,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495484.982, "dur": 9.319,
    "args": {
      "External id": 2480756,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495487.225, "dur": 6.564,
    "args": {
      "External id": 2480757,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495650.623, "dur": 105.700,
    "args": {
      "External id": 2480758,"Sequence number": 24740564, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8461
    }
  },
  {
    "ph": "s", "id": 84, "pid": 1336757, "tid": 1336757, "ts": 1514656495650.623,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495665.641, "dur": 7.742,
    "args": {
      "External id": 2480759,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495667.839, "dur": 5.113,
    "args": {
      "External id": 2480760,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495784.122, "dur": 163.229,
    "args": {
      "External id": 2480761,"Sequence number": 24740565, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8464
    }
  },
  {
    "ph": "s", "id": 83, "pid": 1336757, "tid": 1336757, "ts": 1514656495784.122,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495809.228, "dur": 114.160,
    "args": {
      "External id": 2480762,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495853.509, "dur": 6.487,
    "args": {
      "External id": 2480763,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495855.485, "dur": 4.094,
    "args": {
      "External id": 2480764,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495864.071, "dur": 3.856,
    "args": {
      "External id": 2480765,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495869.273, "dur": 1.671,
    "args": {
      "External id": 2480766,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495873.219, "dur": 3.009,
    "args": {
      "External id": 2480767,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495934.419, "dur": 5.087,
    "args": {
      "External id": 2480768,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495951.585, "dur": 4.839,
    "args": {
      "External id": 2480769,"Sequence number": 24740566, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495952.771, "dur": 3.521,
    "args": {
      "External id": 2480770,"Sequence number": 24740566, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8473
    }
  },
  {
    "ph": "s", "id": 82, "pid": 1336757, "tid": 1336757, "ts": 1514656495952.771,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495966.733, "dur": 147.311,
    "args": {
      "External id": 2480771,"Sequence number": 24740567, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495967.937, "dur": 10.946,
    "args": {
      "External id": 2480772,"Sequence number": 24740567, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8475
    }
  },
  {
    "ph": "s", "id": 81, "pid": 1336757, "tid": 1336757, "ts": 1514656495967.937,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495973.461, "dur": 4.376,
    "args": {
      "External id": 2480773,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495975.957, "dur": 1.627,
    "args": {
      "External id": 2480774,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495979.924, "dur": 133.734,
    "args": {
      "External id": 2480775,"Sequence number": 24740568, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656495981.293, "dur": 42.360,
    "args": {
      "External id": 2480776,"Sequence number": 24740568, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496019.695, "dur": 3.634,
    "args": {
      "External id": 2480777,"Sequence number": 24740568, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8480
    }
  },
  {
    "ph": "s", "id": 80, "pid": 1336757, "tid": 1336757, "ts": 1514656496019.695,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496025.155, "dur": 80.132,
    "args": {
      "External id": 2480778,"Sequence number": 24740569, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8481
    }
  },
  {
    "ph": "s", "id": 79, "pid": 1336757, "tid": 1336757, "ts": 1514656496025.155,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496107.934, "dur": 4.697,
    "args": {
      "External id": 2480779,"Sequence number": 24740570, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8482
    }
  },
  {
    "ph": "s", "id": 78, "pid": 1336757, "tid": 1336757, "ts": 1514656496107.934,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496148.895, "dur": 197.458,
    "args": {
      "External id": 2480780,"Sequence number": 24740571, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8483
    }
  },
  {
    "ph": "s", "id": 77, "pid": 1336757, "tid": 1336757, "ts": 1514656496148.895,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496166.750, "dur": 4.781,
    "args": {
      "External id": 2480781,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496169.476, "dur": 1.880,
    "args": {
      "External id": 2480782,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496175.308, "dur": 3.026,
    "args": {
      "External id": 2480783,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496176.473, "dur": 1.751,
    "args": {
      "External id": 2480784,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496176.919, "dur": 1.168,
    "args": {
      "External id": 2480785,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496185.849, "dur": 6.359,
    "args": {
      "External id": 2480786,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496187.533, "dur": 4.373,
    "args": {
      "External id": 2480787,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496199.912, "dur": 3.247,
    "args": {
      "External id": 2480788,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496206.226, "dur": 3.089,
    "args": {
      "External id": 2480789,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496325.033, "dur": 3.133,
    "args": {
      "External id": 2480790,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496326.099, "dur": 1.887,
    "args": {
      "External id": 2480791,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496330.197, "dur": 2.122,
    "args": {
      "External id": 2480792,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496331.277, "dur": 0.938,
    "args": {
      "External id": 2480793,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496362.214, "dur": 93.416,
    "args": {
      "External id": 2480794,"Sequence number": 24740572, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496369.931, "dur": 7.506,
    "args": {
      "External id": 2480795,"Sequence number": 24740572, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8498
    }
  },
  {
    "ph": "s", "id": 76, "pid": 1336757, "tid": 1336757, "ts": 1514656496369.931,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496372.209, "dur": 4.008,
    "args": {
      "External id": 2480796,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496374.676, "dur": 1.272,
    "args": {
      "External id": 2480797,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496380.341, "dur": 75.006,
    "args": {
      "External id": 2480798,"Sequence number": 24740573, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496382.014, "dur": 4.290,
    "args": {
      "External id": 2480799,"Sequence number": 24740573, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496382.538, "dur": 3.605,
    "args": {
      "External id": 2480800,"Sequence number": 24740573, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8503
    }
  },
  {
    "ph": "s", "id": 75, "pid": 1336757, "tid": 1336757, "ts": 1514656496382.538,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496387.103, "dur": 58.442,
    "args": {
      "External id": 2480801,"Sequence number": 24740574, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8504
    }
  },
  {
    "ph": "s", "id": 74, "pid": 1336757, "tid": 1336757, "ts": 1514656496387.103,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496447.321, "dur": 7.524,
    "args": {
      "External id": 2480802,"Sequence number": 24740575, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8505
    }
  },
  {
    "ph": "s", "id": 73, "pid": 1336757, "tid": 1336757, "ts": 1514656496447.321,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496463.346, "dur": 61.855,
    "args": {
      "External id": 2480803,"Sequence number": 24740576, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496463.803, "dur": 5.629,
    "args": {
      "External id": 2480804,"Sequence number": 24740576, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8507
    }
  },
  {
    "ph": "s", "id": 72, "pid": 1336757, "tid": 1336757, "ts": 1514656496463.803,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496465.358, "dur": 3.027,
    "args": {
      "External id": 2480805,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496467.157, "dur": 1.101,
    "args": {
      "External id": 2480806,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496470.247, "dur": 54.703,
    "args": {
      "External id": 2480807,"Sequence number": 24740577, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496472.714, "dur": 4.073,
    "args": {
      "External id": 2480808,"Sequence number": 24740577, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496473.443, "dur": 3.214,
    "args": {
      "External id": 2480809,"Sequence number": 24740577, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8512
    }
  },
  {
    "ph": "s", "id": 71, "pid": 1336757, "tid": 1336757, "ts": 1514656496473.443,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496477.366, "dur": 42.782,
    "args": {
      "External id": 2480810,"Sequence number": 24740578, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8513
    }
  },
  {
    "ph": "s", "id": 70, "pid": 1336757, "tid": 1336757, "ts": 1514656496477.366,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496522.142, "dur": 2.462,
    "args": {
      "External id": 2480811,"Sequence number": 24740579, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8514
    }
  },
  {
    "ph": "s", "id": 69, "pid": 1336757, "tid": 1336757, "ts": 1514656496522.142,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496565.393, "dur": 167.764,
    "args": {
      "External id": 2480812,"Sequence number": 24740580, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8515
    }
  },
  {
    "ph": "s", "id": 68, "pid": 1336757, "tid": 1336757, "ts": 1514656496565.393,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496601.419, "dur": 4.485,
    "args": {
      "External id": 2480813,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496641.184, "dur": 77.862,
    "args": {
      "External id": 2480814,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496642.399, "dur": 4.914,
    "args": {
      "External id": 2480815,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496643.824, "dur": 2.560,
    "args": {
      "External id": 2480816,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496645.327, "dur": 0.873,
    "args": {
      "External id": 2480817,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496649.748, "dur": 69.019,
    "args": {
      "External id": 2480818,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496651.404, "dur": 2.280,
    "args": {
      "External id": 2480819,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496651.931, "dur": 1.577,
    "args": {
      "External id": 2480820,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496654.202, "dur": 60.674,
    "args": {
      "External id": 2480821,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496717.025, "dur": 1.109,
    "args": {
      "External id": 2480822,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496741.190, "dur": 24.140,
    "args": {
      "External id": 2480823,"Sequence number": 24740581, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8526
    }
  },
  {
    "ph": "s", "id": 67, "pid": 1336757, "tid": 1336757, "ts": 1514656496741.190,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496797.679, "dur": 232.960,
    "args": {
      "External id": 2480824,"Sequence number": 24740582, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [], [], [], [], []], "Ev Idx": 8527
    }
  },
  {
    "ph": "s", "id": 66, "pid": 1336757, "tid": 1336757, "ts": 1514656496797.679,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496819.097, "dur": 2.966,
    "args": {
      "External id": 2480825,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496820.085, "dur": 1.824,
    "args": {
      "External id": 2480826,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496832.169, "dur": 10.948,
    "args": {
      "External id": 2480827,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496838.424, "dur": 4.121,
    "args": {
      "External id": 2480828,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496851.981, "dur": 4.869,
    "args": {
      "External id": 2480829,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496976.357, "dur": 2.796,
    "args": {
      "External id": 2480830,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656496977.289, "dur": 1.701,
    "args": {
      "External id": 2480831,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497050.990, "dur": 96.503,
    "args": {
      "External id": 2480832,"Sequence number": 24740583, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497052.274, "dur": 7.952,
    "args": {
      "External id": 2480833,"Sequence number": 24740583, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8536
    }
  },
  {
    "ph": "s", "id": 65, "pid": 1336757, "tid": 1336757, "ts": 1514656497052.274,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497054.664, "dur": 4.090,
    "args": {
      "External id": 2480834,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497056.826, "dur": 1.658,
    "args": {
      "External id": 2480835,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497061.257, "dur": 85.851,
    "args": {
      "External id": 2480836,"Sequence number": 24740584, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497064.359, "dur": 4.044,
    "args": {
      "External id": 2480837,"Sequence number": 24740584, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497065.137, "dur": 3.084,
    "args": {
      "External id": 2480838,"Sequence number": 24740584, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8541
    }
  },
  {
    "ph": "s", "id": 64, "pid": 1336757, "tid": 1336757, "ts": 1514656497065.137,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497069.112, "dur": 71.637,
    "args": {
      "External id": 2480839,"Sequence number": 24740585, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8542
    }
  },
  {
    "ph": "s", "id": 63, "pid": 1336757, "tid": 1336757, "ts": 1514656497069.112,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497142.807, "dur": 3.376,
    "args": {
      "External id": 2480840,"Sequence number": 24740586, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8543
    }
  },
  {
    "ph": "s", "id": 62, "pid": 1336757, "tid": 1336757, "ts": 1514656497142.807,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497155.206, "dur": 64.155,
    "args": {
      "External id": 2480841,"Sequence number": 24740587, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497158.068, "dur": 7.162,
    "args": {
      "External id": 2480842,"Sequence number": 24740587, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8545
    }
  },
  {
    "ph": "s", "id": 61, "pid": 1336757, "tid": 1336757, "ts": 1514656497158.068,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497159.997, "dur": 4.083,
    "args": {
      "External id": 2480843,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497161.435, "dur": 2.421,
    "args": {
      "External id": 2480844,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497165.793, "dur": 53.347,
    "args": {
      "External id": 2480845,"Sequence number": 24740588, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497166.925, "dur": 5.234,
    "args": {
      "External id": 2480846,"Sequence number": 24740588, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497169.336, "dur": 2.557,
    "args": {
      "External id": 2480847,"Sequence number": 24740588, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8550
    }
  },
  {
    "ph": "s", "id": 60, "pid": 1336757, "tid": 1336757, "ts": 1514656497169.336,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497172.859, "dur": 39.512,
    "args": {
      "External id": 2480848,"Sequence number": 24740589, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8551
    }
  },
  {
    "ph": "s", "id": 59, "pid": 1336757, "tid": 1336757, "ts": 1514656497172.859,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497214.087, "dur": 4.581,
    "args": {
      "External id": 2480849,"Sequence number": 24740590, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8552
    }
  },
  {
    "ph": "s", "id": 58, "pid": 1336757, "tid": 1336757, "ts": 1514656497214.087,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497226.978, "dur": 57.581,
    "args": {
      "External id": 2480850,"Sequence number": 24740591, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497227.511, "dur": 6.186,
    "args": {
      "External id": 2480851,"Sequence number": 24740591, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8554
    }
  },
  {
    "ph": "s", "id": 57, "pid": 1336757, "tid": 1336757, "ts": 1514656497227.511,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497230.351, "dur": 2.285,
    "args": {
      "External id": 2480852,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497231.753, "dur": 0.744,
    "args": {
      "External id": 2480853,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497234.242, "dur": 50.113,
    "args": {
      "External id": 2480854,"Sequence number": 24740592, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497235.055, "dur": 6.101,
    "args": {
      "External id": 2480855,"Sequence number": 24740592, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497237.732, "dur": 3.310,
    "args": {
      "External id": 2480856,"Sequence number": 24740592, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8559
    }
  },
  {
    "ph": "s", "id": 56, "pid": 1336757, "tid": 1336757, "ts": 1514656497237.732,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497241.586, "dur": 38.505,
    "args": {
      "External id": 2480857,"Sequence number": 24740593, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8560
    }
  },
  {
    "ph": "s", "id": 55, "pid": 1336757, "tid": 1336757, "ts": 1514656497241.586,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497281.820, "dur": 2.186,
    "args": {
      "External id": 2480858,"Sequence number": 24740594, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8561
    }
  },
  {
    "ph": "s", "id": 54, "pid": 1336757, "tid": 1336757, "ts": 1514656497281.820,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497299.697, "dur": 5.058,
    "args": {
      "External id": 2480859,"Sequence number": 24740595, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497300.760, "dur": 3.852,
    "args": {
      "External id": 2480860,"Sequence number": 24740595, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8563
    }
  },
  {
    "ph": "s", "id": 53, "pid": 1336757, "tid": 1336757, "ts": 1514656497300.760,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497311.027, "dur": 3.177,
    "args": {
      "External id": 2480861,"Sequence number": 24740596, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497312.062, "dur": 2.022,
    "args": {
      "External id": 2480862,"Sequence number": 24740596, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8565
    }
  },
  {
    "ph": "s", "id": 52, "pid": 1336757, "tid": 1336757, "ts": 1514656497312.062,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497317.834, "dur": 5.250,
    "args": {
      "External id": 2480863,"Sequence number": 24740597, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497318.909, "dur": 4.015,
    "args": {
      "External id": 2480864,"Sequence number": 24740597, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 32, 64]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8567
    }
  },
  {
    "ph": "s", "id": 51, "pid": 1336757, "tid": 1336757, "ts": 1514656497318.909,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497350.816, "dur": 155.763,
    "args": {
      "External id": 2480865,"Sequence number": 24740598, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8568
    }
  },
  {
    "ph": "s", "id": 50, "pid": 1336757, "tid": 1336757, "ts": 1514656497350.816,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497370.931, "dur": 7.896,
    "args": {
      "External id": 2480866,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497373.353, "dur": 5.029,
    "args": {
      "External id": 2480867,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "RotaryEmbeddingFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497519.084, "dur": 129.156,
    "args": {
      "External id": 2480868,"Sequence number": 24740599, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "False", "False", "0", "", "8192"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [32, 1], [32, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [8192, 32], [8192, 32], [], [], [], [], []], "Ev Idx": 8571
    }
  },
  {
    "ph": "s", "id": 49, "pid": 1336757, "tid": 1336757, "ts": 1514656497519.084,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497552.282, "dur": 8.784,
    "args": {
      "External id": 2480869,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497554.686, "dur": 5.880,
    "args": {
      "External id": 2480870,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FlashAttnFunc", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497677.934, "dur": 173.535,
    "args": {
      "External id": 2480871,"Sequence number": 24740600, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "", "True", "", "0.", "", "False", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "", "Scalar", "", "Scalar", "", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], [], []], "Ev Idx": 8574
    }
  },
  {
    "ph": "s", "id": 48, "pid": 1336757, "tid": 1336757, "ts": 1514656497677.934,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "flash_attn::_flash_attn_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497703.549, "dur": 121.931,
    "args": {
      "External id": 2480872,"Record function id": 0, "Concrete Inputs": ["", "", "", "0.", "0.125", "True", "-1", "-1", "0.", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [8388608, 2048, 64, 1], [], [], [], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [16, 4096, 32, 64], [16, 4096, 32, 64], [], [], [], [], [], [], [], []], "Ev Idx": 8575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497752.224, "dur": 7.324,
    "args": {
      "External id": 2480873,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", ""], "Input type": ["c10::BFloat16", "", "", "", "", ""], "Input Strides": [[8388608, 2048, 64, 1], [], [], [], [], []], "Input Dims": [[16, 4096, 32, 64], [], [], [], [], []], "Ev Idx": 8576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497754.439, "dur": 4.673,
    "args": {
      "External id": 2480874,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 32, 64]", "[8388608, 2048, 64, 1]", "15", "0", "", ""], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497762.078, "dur": 3.995,
    "args": {
      "External id": 2480875,"Record function id": 0, "Concrete Inputs": ["[16, 32, 4096]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497767.223, "dur": 3.109,
    "args": {
      "External id": 2480876,"Record function id": 0, "Concrete Inputs": ["[0]", "15", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497772.792, "dur": 2.849,
    "args": {
      "External id": 2480877,"Record function id": 0, "Concrete Inputs": ["[2]", "4", "", "", "", ""], "Input type": ["ScalarList", "Scalar", "", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::alias", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497837.806, "dur": 4.979,
    "args": {
      "External id": 2480878,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[8388608, 2048, 64, 1]], "Input Dims": [[16, 4096, 32, 64]], "Ev Idx": 8581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497856.420, "dur": 5.691,
    "args": {
      "External id": 2480879,"Sequence number": 24740601, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497857.536, "dur": 4.369,
    "args": {
      "External id": 2480880,"Sequence number": 24740601, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, -1]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 64, 1], []], "Input Dims": [[16, 4096, 32, 64], []], "Ev Idx": 8583
    }
  },
  {
    "ph": "s", "id": 47, "pid": 1336757, "tid": 1336757, "ts": 1514656497857.536,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497873.223, "dur": 98.602,
    "args": {
      "External id": 2480881,"Sequence number": 24740602, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [2048, 2048], []], "Ev Idx": 8584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497874.454, "dur": 8.999,
    "args": {
      "External id": 2480882,"Sequence number": 24740602, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[2048, 2048]], "Ev Idx": 8585
    }
  },
  {
    "ph": "s", "id": 46, "pid": 1336757, "tid": 1336757, "ts": 1514656497874.454,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497876.572, "dur": 5.678,
    "args": {
      "External id": 2480883,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[2048, 2048], [], []], "Ev Idx": 8586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497880.557, "dur": 1.479,
    "args": {
      "External id": 2480884,"Record function id": 0, "Concrete Inputs": ["", "[2048, 2048]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[2048, 2048], [], [], []], "Ev Idx": 8587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497884.654, "dur": 86.877,
    "args": {
      "External id": 2480885,"Sequence number": 24740603, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 2048]], "Ev Idx": 8588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497886.443, "dur": 3.413,
    "args": {
      "External id": 2480886,"Sequence number": 24740603, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497887.411, "dur": 2.254,
    "args": {
      "External id": 2480887,"Sequence number": 24740603, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8590
    }
  },
  {
    "ph": "s", "id": 45, "pid": 1336757, "tid": 1336757, "ts": 1514656497887.411,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497890.636, "dur": 74.370,
    "args": {
      "External id": 2480888,"Sequence number": 24740604, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 2048]], "Ev Idx": 8591
    }
  },
  {
    "ph": "s", "id": 44, "pid": 1336757, "tid": 1336757, "ts": 1514656497890.636,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656497967.011, "dur": 3.755,
    "args": {
      "External id": 2480889,"Sequence number": 24740605, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8592
    }
  },
  {
    "ph": "s", "id": 43, "pid": 1336757, "tid": 1336757, "ts": 1514656497967.011,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "LayerNormFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498044.696, "dur": 205.586,
    "args": {
      "External id": 2480890,"Sequence number": 24740606, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "9.9999999999999995e-07", "True", "False", "True"], "Input type": ["c10::BFloat16", "c10::BFloat16", "", "c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8388608, 2048, 1], [1], [], [8388608, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [2048], [], [16, 4096, 2048], [], [], [], []], "Ev Idx": 8593
    }
  },
  {
    "ph": "s", "id": 42, "pid": 1336757, "tid": 1336757, "ts": 1514656498044.696,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498063.973, "dur": 3.902,
    "args": {
      "External id": 2480891,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498064.675, "dur": 2.907,
    "args": {
      "External id": 2480892,"Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498072.143, "dur": 4.564,
    "args": {
      "External id": 2480893,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [2048, 1]], "Input Dims": [[16, 4096, 2048], [65536, 2048]], "Ev Idx": 8596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498074.626, "dur": 1.975,
    "args": {
      "External id": 2480894,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498075.336, "dur": 1.110,
    "args": {
      "External id": 2480895,"Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498083.961, "dur": 7.504,
    "args": {
      "External id": 2480896,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498086.043, "dur": 5.092,
    "args": {
      "External id": 2480897,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498097.509, "dur": 3.177,
    "args": {
      "External id": 2480898,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498104.482, "dur": 4.161,
    "args": {
      "External id": 2480899,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498230.346, "dur": 3.327,
    "args": {
      "External id": 2480900,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498231.300, "dur": 2.118,
    "args": {
      "External id": 2480901,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498236.204, "dur": 2.165,
    "args": {
      "External id": 2480902,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498237.260, "dur": 1.008,
    "args": {
      "External id": 2480903,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498266.965, "dur": 95.340,
    "args": {
      "External id": 2480904,"Sequence number": 24740607, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498268.237, "dur": 10.345,
    "args": {
      "External id": 2480905,"Sequence number": 24740607, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8608
    }
  },
  {
    "ph": "s", "id": 41, "pid": 1336757, "tid": 1336757, "ts": 1514656498268.237,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498270.491, "dur": 6.878,
    "args": {
      "External id": 2480906,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498274.408, "dur": 2.691,
    "args": {
      "External id": 2480907,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498279.610, "dur": 82.350,
    "args": {
      "External id": 2480908,"Sequence number": 24740608, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498281.180, "dur": 3.655,
    "args": {
      "External id": 2480909,"Sequence number": 24740608, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498281.797, "dur": 2.773,
    "args": {
      "External id": 2480910,"Sequence number": 24740608, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8613
    }
  },
  {
    "ph": "s", "id": 40, "pid": 1336757, "tid": 1336757, "ts": 1514656498281.797,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498287.184, "dur": 66.708,
    "args": {
      "External id": 2480911,"Sequence number": 24740609, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8614
    }
  },
  {
    "ph": "s", "id": 39, "pid": 1336757, "tid": 1336757, "ts": 1514656498287.184,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498356.042, "dur": 5.353,
    "args": {
      "External id": 2480912,"Sequence number": 24740610, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8615
    }
  },
  {
    "ph": "s", "id": 38, "pid": 1336757, "tid": 1336757, "ts": 1514656498356.042,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498370.353, "dur": 68.437,
    "args": {
      "External id": 2480913,"Sequence number": 24740611, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[8388608, 2048, 1], [2048, 1], []], "Input Dims": [[16, 4096, 2048], [5632, 2048], []], "Ev Idx": 8616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498370.956, "dur": 8.685,
    "args": {
      "External id": 2480914,"Sequence number": 24740611, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[5632, 2048]], "Ev Idx": 8617
    }
  },
  {
    "ph": "s", "id": 37, "pid": 1336757, "tid": 1336757, "ts": 1514656498370.956,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498374.568, "dur": 4.105,
    "args": {
      "External id": 2480915,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[5632, 2048], [], []], "Ev Idx": 8618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498377.719, "dur": 0.777,
    "args": {
      "External id": 2480916,"Record function id": 0, "Concrete Inputs": ["", "[2048, 5632]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[5632, 2048], [], [], []], "Ev Idx": 8619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498380.367, "dur": 58.203,
    "args": {
      "External id": 2480917,"Sequence number": 24740612, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[8388608, 2048, 1], [1, 2048]], "Input Dims": [[16, 4096, 2048], [2048, 5632]], "Ev Idx": 8620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498381.524, "dur": 5.103,
    "args": {
      "External id": 2480918,"Sequence number": 24740612, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498382.485, "dur": 4.034,
    "args": {
      "External id": 2480919,"Sequence number": 24740612, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[65536, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8622
    }
  },
  {
    "ph": "s", "id": 36, "pid": 1336757, "tid": 1336757, "ts": 1514656498382.485,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498387.228, "dur": 46.689,
    "args": {
      "External id": 2480920,"Sequence number": 24740613, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[65536, 2048], [2048, 5632]], "Ev Idx": 8623
    }
  },
  {
    "ph": "s", "id": 35, "pid": 1336757, "tid": 1336757, "ts": 1514656498387.228,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498435.856, "dur": 2.434,
    "args": {
      "External id": 2480921,"Sequence number": 24740614, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[5632, 1], []], "Input Dims": [[65536, 5632], []], "Ev Idx": 8624
    }
  },
  {
    "ph": "s", "id": 34, "pid": 1336757, "tid": 1336757, "ts": 1514656498435.856,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "SwiGLULinearFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498457.682, "dur": 161.130,
    "args": {
      "External id": 2480922,"Sequence number": 24740615, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8625
    }
  },
  {
    "ph": "s", "id": 33, "pid": 1336757, "tid": 1336757, "ts": 1514656498457.682,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498493.381, "dur": 3.642,
    "args": {
      "External id": 2480923,"Record function id": 0, "Concrete Inputs": ["[16, 4096, 5632]", "15", "", "", "", "0"], "Input type": ["ScalarList", "Scalar", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498526.374, "dur": 80.246,
    "args": {
      "External id": 2480924,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[23068672, 5632, 1], [5632, 1], []], "Input Dims": [[16, 4096, 5632], [2048, 5632], []], "Ev Idx": 8627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498544.274, "dur": 8.310,
    "args": {
      "External id": 2480925,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[5632, 1]], "Input Dims": [[2048, 5632]], "Ev Idx": 8628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498545.916, "dur": 5.620,
    "args": {
      "External id": 2480926,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[5632, 1], [], []], "Input Dims": [[2048, 5632], [], []], "Ev Idx": 8629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498548.243, "dur": 2.905,
    "args": {
      "External id": 2480927,"Record function id": 0, "Concrete Inputs": ["", "[5632, 2048]", "[1, 5632]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[5632, 1], [], [], []], "Input Dims": [[2048, 5632], [], [], []], "Ev Idx": 8630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498553.378, "dur": 52.835,
    "args": {
      "External id": 2480928,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[23068672, 5632, 1], [1, 5632]], "Input Dims": [[16, 4096, 5632], [5632, 2048]], "Ev Idx": 8631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reshape", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498554.931, "dur": 2.284,
    "args": {
      "External id": 2480929,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498555.571, "dur": 1.360,
    "args": {
      "External id": 2480930,"Record function id": 0, "Concrete Inputs": ["", "[65536, 5632]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[23068672, 5632, 1], []], "Input Dims": [[16, 4096, 5632], []], "Ev Idx": 8633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498557.830, "dur": 44.245,
    "args": {
      "External id": 2480931,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[5632, 1], [1, 5632]], "Input Dims": [[65536, 5632], [5632, 2048]], "Ev Idx": 8634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_unsafe_view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498604.244, "dur": 1.337,
    "args": {
      "External id": 2480932,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498627.745, "dur": 24.048,
    "args": {
      "External id": 2480933,"Sequence number": 24740616, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[8388608, 2048, 1], [8388608, 2048, 1], []], "Input Dims": [[16, 4096, 2048], [16, 4096, 2048], []], "Ev Idx": 8636
    }
  },
  {
    "ph": "s", "id": 32, "pid": 1336757, "tid": 1336757, "ts": 1514656498627.745,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::stack", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498672.720, "dur": 38.093,
    "args": {
      "External id": 2480934,"Sequence number": 24740617, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "-2"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[[8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1]], []], "Input Dims": [[[16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048]], []], "Ev Idx": 8637
    }
  },
  {
    "ph": "s", "id": 31, "pid": 1336757, "tid": 1336757, "ts": 1514656498672.720,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::cat", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498680.781, "dur": 25.516,
    "args": {
      "External id": 2480935,"Record function id": 0, "Concrete Inputs": ["", "2"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[[8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1], [8388608, 2048, 1]], []], "Input Dims": [[[16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048], [16, 4096, 2048]], []], "Ev Idx": 8638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498707.715, "dur": 1.548,
    "args": {
      "External id": 2480936,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 8192], []], "Ev Idx": 8639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "TorchDynamo Cache Lookup", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498744.548, "dur": 33.312,
    "args": {
      "External id": 2480937,"Record function id": 0, "Ev Idx": 8640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Torch-Compiled Region: 2/0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498778.867, "dur": 184.004,
    "args": {
      "External id": 2480938,"Record function id": 0, "Ev Idx": 8641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "CompiledFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498814.712, "dur": 139.239,
    "args": {
      "External id": 2480939,"Sequence number": 24740618, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1], [33554432, 8192, 2048, 1]], "Input Dims": [[2048], [16, 4096, 4, 2048]], "Ev Idx": 8642
    }
  },
  {
    "ph": "s", "id": 30, "pid": 1336757, "tid": 1336757, "ts": 1514656498814.712,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "layer_norm_fwd_kernel_0", "pid": 1336757, "tid": 1336757,
    "ts": 1514656498877.512, "dur": 40.888,
    "args": {
      "External id": 2480940,"kernel_hash": "cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh", "Record function id": 0, "stream": 0, "Concrete Inputs": ["", "", "", "", "2048", "1", "9.9999999999999995e-07", "True", "2048", "False", "False", "True", "False"], "kernel_file": "/tmp/torchinductor_cvm/fm/cfm353rsfwkbkca5jmv2fnid3xh6d36yeu3b4ybp4yeabcp47toh.py", "kernel_backend": "triton", "Input type": ["c10::BFloat16", "c10::BFloat16", "c10::BFloat16", "float", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], [1], [1], [], [], [], [], [], [], [], [], []], "Input Dims": [[262144, 2048], [262144, 2048], [2048], [262144], [], [], [], [], [], [], [], [], []], "Ev Idx": 8643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499091.328, "dur": 43.405,
    "args": {
      "External id": 2480941,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False"], "Input type": ["ScalarList", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499094.366, "dur": 6.106,
    "args": {
      "External id": 2480942,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False", ""], "Input type": ["ScalarList", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499104.417, "dur": 29.976,
    "args": {
      "External id": 2480943,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 8646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499108.112, "dur": 25.483,
    "args": {
      "External id": 2480944,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 8647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499139.574, "dur": 16.447,
    "args": {
      "External id": 2480945,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False"], "Input type": ["ScalarList", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499140.328, "dur": 2.997,
    "args": {
      "External id": 2480946,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False", ""], "Input type": ["ScalarList", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499144.046, "dur": 11.661,
    "args": {
      "External id": 2480947,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 8650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499144.548, "dur": 10.722,
    "args": {
      "External id": 2480948,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 8651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499158.908, "dur": 16.484,
    "args": {
      "External id": 2480949,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False"], "Input type": ["ScalarList", "", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499161.397, "dur": 2.179,
    "args": {
      "External id": 2480950,"Record function id": 0, "Concrete Inputs": ["[1]", "", "", "", "False", ""], "Input type": ["ScalarList", "", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499164.111, "dur": 11.006,
    "args": {
      "External id": 2480951,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[1]], "Ev Idx": 8654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499164.874, "dur": 9.858,
    "args": {
      "External id": 2480952,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 8655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499185.360, "dur": 0.693,
    "args": {
      "External id": 2480953,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "False", "False", ""], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[8192, 1], [], [], [], [], [], [], []], "Input Dims": [[16, 8192], [], [], [], [], [], [], []], "Ev Idx": 8656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unfold", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499195.050, "dur": 10.120,
    "args": {
      "External id": 2480954,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "5", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 8657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499201.305, "dur": 1.939,
    "args": {
      "External id": 2480955,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 5]", "[8192, 1, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 8192], [], [], []], "Ev Idx": 8658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499211.875, "dur": 7.025,
    "args": {
      "External id": 2480956,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 5], [], [], [], []], "Ev Idx": 8659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499216.030, "dur": 1.162,
    "args": {
      "External id": 2480957,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 5]", "[8192, 1, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 5], [], [], []], "Ev Idx": 8660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499220.065, "dur": 3.229,
    "args": {
      "External id": 2480958,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 5], [], [], [], []], "Ev Idx": 8661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499221.874, "dur": 0.733,
    "args": {
      "External id": 2480959,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 5]", "[8192, 1, 1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 5], [], [], []], "Ev Idx": 8662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499224.776, "dur": 2.584,
    "args": {
      "External id": 2480960,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "1", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 5], [], [], [], []], "Ev Idx": 8663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499226.270, "dur": 0.548,
    "args": {
      "External id": 2480961,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 5], [], [], []], "Ev Idx": 8664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499231.881, "dur": 2.948,
    "args": {
      "External id": 2480962,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 4], [], [], [], []], "Ev Idx": 8665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499233.477, "dur": 0.753,
    "args": {
      "External id": 2480963,"Record function id": 0, "Concrete Inputs": ["", "[16, 8188, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 4], [], [], []], "Ev Idx": 8666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499235.818, "dur": 3.152,
    "args": {
      "External id": 2480964,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "4096", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 8188, 4], [], [], [], []], "Ev Idx": 8667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499237.474, "dur": 0.750,
    "args": {
      "External id": 2480965,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 8188, 4], [], [], []], "Ev Idx": 8668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499239.939, "dur": 3.224,
    "args": {
      "External id": 2480966,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4096, 4], [], [], [], []], "Ev Idx": 8669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499241.793, "dur": 0.778,
    "args": {
      "External id": 2480967,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4096, 4], [], [], []], "Ev Idx": 8670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499248.002, "dur": 5.201,
    "args": {
      "External id": 2480968,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "2"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4096, 4], [], []], "Ev Idx": 8671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499251.635, "dur": 0.668,
    "args": {
      "External id": 2480969,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4096, 4], [], [], []], "Ev Idx": 8672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499257.016, "dur": 3.055,
    "args": {
      "External id": 2480970,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 8673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499258.721, "dur": 0.743,
    "args": {
      "External id": 2480971,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 8674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499264.582, "dur": 9.429,
    "args": {
      "External id": 2480972,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 8675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499272.058, "dur": 0.729,
    "args": {
      "External id": 2480973,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 8676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499277.065, "dur": 2.665,
    "args": {
      "External id": 2480974,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 8677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499278.417, "dur": 0.768,
    "args": {
      "External id": 2480975,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 8678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499282.040, "dur": 6.095,
    "args": {
      "External id": 2480976,"Sequence number": 24740619, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 8679
    }
  },
  {
    "ph": "s", "id": 29, "pid": 1336757, "tid": 1336757, "ts": 1514656499282.040,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499285.871, "dur": 0.664,
    "args": {
      "External id": 2480977,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 8680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499289.095, "dur": 6.490,
    "args": {
      "External id": 2480978,"Sequence number": 24740620, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 8681
    }
  },
  {
    "ph": "s", "id": 28, "pid": 1336757, "tid": 1336757, "ts": 1514656499289.095,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499294.028, "dur": 0.739,
    "args": {
      "External id": 2480979,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 8682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499296.562, "dur": 4.677,
    "args": {
      "External id": 2480980,"Sequence number": 24740621, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 8683
    }
  },
  {
    "ph": "s", "id": 27, "pid": 1336757, "tid": 1336757, "ts": 1514656499296.562,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499299.488, "dur": 0.912,
    "args": {
      "External id": 2480981,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 8684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499302.244, "dur": 5.998,
    "args": {
      "External id": 2480982,"Sequence number": 24740622, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 8685
    }
  },
  {
    "ph": "s", "id": 26, "pid": 1336757, "tid": 1336757, "ts": 1514656499302.244,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499305.198, "dur": 2.231,
    "args": {
      "External id": 2480983,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 8686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499313.311, "dur": 37.601,
    "args": {
      "External id": 2480984,"Sequence number": 24740623, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 8687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499315.711, "dur": 34.967,
    "args": {
      "External id": 2480985,"Sequence number": 24740623, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 8688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499318.160, "dur": 7.675,
    "args": {
      "External id": 2480986,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 8689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499322.113, "dur": 3.146,
    "args": {
      "External id": 2480987,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499327.287, "dur": 22.885,
    "args": {
      "External id": 2480988,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 8691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499375.486, "dur": 4.247,
    "args": {
      "External id": 2480989,"Sequence number": 24740623, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 8692
    }
  },
  {
    "ph": "s", "id": 25, "pid": 1336757, "tid": 1336757, "ts": 1514656499375.486,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499381.962, "dur": 1.185,
    "args": {
      "External id": 2480990,"Sequence number": 24740624, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 8693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499410.130, "dur": 72234.308,
    "args": {
      "External id": 2480991,"Sequence number": 24740624, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 8694
    }
  },
  {
    "ph": "s", "id": 24, "pid": 1336757, "tid": 1336757, "ts": 1514656499410.130,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499424.623, "dur": 29.175,
    "args": {
      "External id": 2480992,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499425.363, "dur": 28.185,
    "args": {
      "External id": 2480993,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499426.563, "dur": 8.034,
    "args": {
      "External id": 2480994,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499429.830, "dur": 4.410,
    "args": {
      "External id": 2480995,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499435.487, "dur": 17.411,
    "args": {
      "External id": 2480996,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 8699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499471.567, "dur": 26.733,
    "args": {
      "External id": 2480997,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499473.632, "dur": 6.486,
    "args": {
      "External id": 2480998,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 8701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499475.813, "dur": 3.991,
    "args": {
      "External id": 2480999,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499480.862, "dur": 17.239,
    "args": {
      "External id": 2481000,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 8703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499481.965, "dur": 15.707,
    "args": {
      "External id": 2481001,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 8704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499501.524, "dur": 44.367,
    "args": {
      "External id": 2481002,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 8705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499502.390, "dur": 4.403,
    "args": {
      "External id": 2481003,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 8706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499503.492, "dur": 3.059,
    "args": {
      "External id": 2481004,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499514.849, "dur": 30.820,
    "args": {
      "External id": 2481005,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499517.250, "dur": 27.710,
    "args": {
      "External id": 2481006,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 8709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499556.507, "dur": 20.261,
    "args": {
      "External id": 2481007,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 8710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499557.869, "dur": 5.315,
    "args": {
      "External id": 2481008,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499564.075, "dur": 12.431,
    "args": {
      "External id": 2481009,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 8712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499564.640, "dur": 11.463,
    "args": {
      "External id": 2481010,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 8713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499584.238, "dur": 29.447,
    "args": {
      "External id": 2481011,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 8714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499619.054, "dur": 64.922,
    "args": {
      "External id": 2481012,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 8715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499624.493, "dur": 59.087,
    "args": {
      "External id": 2481013,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499634.538, "dur": 1.324,
    "args": {
      "External id": 2481014,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 8717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499639.553, "dur": 25.187,
    "args": {
      "External id": 2481015,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499641.464, "dur": 23.069,
    "args": {
      "External id": 2481016,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 8719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499645.992, "dur": 2.836,
    "args": {
      "External id": 2481017,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499649.668, "dur": 14.357,
    "args": {
      "External id": 2481018,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 8721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499689.453, "dur": 66083.559,
    "args": {
      "External id": 2481019,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 8722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336757, "tid": 1336757,
    "ts": 1514656499691.738, "dur": 66079.710,
    "args": {
      "External id": 2481020,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 8723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656565788.736, "dur": 8.427,
    "args": {
      "External id": 2481021,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656565793.764, "dur": 1.268,
    "args": {
      "External id": 2481022,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656565803.238, "dur": 118.529,
    "args": {
      "External id": 2481023,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656565807.006, "dur": 9.498,
    "args": {
      "External id": 2481024,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656565809.555, "dur": 6.142,
    "args": {
      "External id": 2481025,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656565814.705, "dur": 0.705,
    "args": {
      "External id": 2481026,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656565817.862, "dur": 103.086,
    "args": {
      "External id": 2481027,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656565819.957, "dur": 100.070,
    "args": {
      "External id": 2481028,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656565925.825, "dur": 4.555,
    "args": {
      "External id": 2481029,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656565928.056, "dur": 0.939,
    "args": {
      "External id": 2481030,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656565942.049, "dur": 4.022,
    "args": {
      "External id": 2481031,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656565959.280, "dur": 7.321,
    "args": {
      "External id": 2481032,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656565962.219, "dur": 4.130,
    "args": {
      "External id": 2481033,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566145.141, "dur": 258.690,
    "args": {
      "External id": 2481034,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566152.130, "dur": 3.319,
    "args": {
      "External id": 2481035,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566158.873, "dur": 244.103,
    "args": {
      "External id": 2481036,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566165.226, "dur": 1.089,
    "args": {
      "External id": 2481037,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566168.709, "dur": 33.834,
    "args": {
      "External id": 2481038,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566205.598, "dur": 6.680,
    "args": {
      "External id": 2481039,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566210.686, "dur": 1.120,
    "args": {
      "External id": 2481040,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566214.321, "dur": 26.027,
    "args": {
      "External id": 2481041,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566215.637, "dur": 1.510,
    "args": {
      "External id": 2481042,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566219.514, "dur": 20.517,
    "args": {
      "External id": 2481043,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566224.500, "dur": 3.313,
    "args": {
      "External id": 2481044,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566243.896, "dur": 24.854,
    "args": {
      "External id": 2481045,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566272.168, "dur": 21.845,
    "args": {
      "External id": 2481046,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566300.022, "dur": 19.727,
    "args": {
      "External id": 2481047,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566322.998, "dur": 17.258,
    "args": {
      "External id": 2481048,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566343.231, "dur": 22.241,
    "args": {
      "External id": 2481049,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566345.571, "dur": 1.711,
    "args": {
      "External id": 2481050,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566350.016, "dur": 0.628,
    "args": {
      "External id": 2481051,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566368.958, "dur": 17.252,
    "args": {
      "External id": 2481052,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566390.355, "dur": 11.353,
    "args": {
      "External id": 2481053,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566411.531, "dur": 2.962,
    "args": {
      "External id": 2481054,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566421.835, "dur": 4.665,
    "args": {
      "External id": 2481055,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566424.794, "dur": 0.788,
    "args": {
      "External id": 2481056,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566513.751, "dur": 93.308,
    "args": {
      "External id": 2481057,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566613.757, "dur": 6.280,
    "args": {
      "External id": 2481058,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566617.126, "dur": 1.195,
    "args": {
      "External id": 2481059,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566622.363, "dur": 32.521,
    "args": {
      "External id": 2481060,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566664.082, "dur": 6.471,
    "args": {
      "External id": 2481061,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566665.944, "dur": 3.678,
    "args": {
      "External id": 2481062,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566667.889, "dur": 1.451,
    "args": {
      "External id": 2481063,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566675.635, "dur": 50.135,
    "args": {
      "External id": 2481064,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566677.332, "dur": 47.847,
    "args": {
      "External id": 2481065,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566732.101, "dur": 18.723,
    "args": {
      "External id": 2481066,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566756.968, "dur": 5.224,
    "args": {
      "External id": 2481067,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566760.609, "dur": 0.778,
    "args": {
      "External id": 2481068,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566766.629, "dur": 51.395,
    "args": {
      "External id": 2481069,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566767.822, "dur": 4.192,
    "args": {
      "External id": 2481070,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566768.886, "dur": 2.573,
    "args": {
      "External id": 2481071,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566770.421, "dur": 0.905,
    "args": {
      "External id": 2481072,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566772.889, "dur": 44.668,
    "args": {
      "External id": 2481073,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566775.257, "dur": 41.783,
    "args": {
      "External id": 2481074,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566822.218, "dur": 3.519,
    "args": {
      "External id": 2481075,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566824.143, "dur": 0.614,
    "args": {
      "External id": 2481076,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566832.282, "dur": 1.917,
    "args": {
      "External id": 2481077,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566842.532, "dur": 9.518,
    "args": {
      "External id": 2481078,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566846.432, "dur": 5.312,
    "args": {
      "External id": 2481079,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566949.649, "dur": 222.073,
    "args": {
      "External id": 2481080,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566952.021, "dur": 2.544,
    "args": {
      "External id": 2481081,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566955.997, "dur": 215.203,
    "args": {
      "External id": 2481082,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566957.814, "dur": 0.290,
    "args": {
      "External id": 2481083,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656566959.522, "dur": 57.093,
    "args": {
      "External id": 2481084,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567019.778, "dur": 3.574,
    "args": {
      "External id": 2481085,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567022.076, "dur": 0.960,
    "args": {
      "External id": 2481086,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567024.458, "dur": 27.968,
    "args": {
      "External id": 2481087,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567027.124, "dur": 2.466,
    "args": {
      "External id": 2481088,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567030.839, "dur": 21.251,
    "args": {
      "External id": 2481089,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567036.260, "dur": 2.805,
    "args": {
      "External id": 2481090,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567054.122, "dur": 21.580,
    "args": {
      "External id": 2481091,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567077.243, "dur": 12.981,
    "args": {
      "External id": 2481092,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567093.555, "dur": 14.296,
    "args": {
      "External id": 2481093,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567109.504, "dur": 12.039,
    "args": {
      "External id": 2481094,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567123.307, "dur": 21.718,
    "args": {
      "External id": 2481095,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567125.459, "dur": 1.559,
    "args": {
      "External id": 2481096,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567130.690, "dur": 0.704,
    "args": {
      "External id": 2481097,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567146.570, "dur": 11.775,
    "args": {
      "External id": 2481098,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567159.456, "dur": 10.628,
    "args": {
      "External id": 2481099,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567179.530, "dur": 2.569,
    "args": {
      "External id": 2481100,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567191.521, "dur": 3.947,
    "args": {
      "External id": 2481101,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567194.045, "dur": 0.599,
    "args": {
      "External id": 2481102,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567269.216, "dur": 56.739,
    "args": {
      "External id": 2481103,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567330.771, "dur": 6.555,
    "args": {
      "External id": 2481104,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567335.142, "dur": 0.941,
    "args": {
      "External id": 2481105,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567338.863, "dur": 26.002,
    "args": {
      "External id": 2481106,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567369.483, "dur": 5.051,
    "args": {
      "External id": 2481107,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567370.834, "dur": 3.033,
    "args": {
      "External id": 2481108,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567372.427, "dur": 1.233,
    "args": {
      "External id": 2481109,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567377.161, "dur": 45.877,
    "args": {
      "External id": 2481110,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567380.487, "dur": 41.999,
    "args": {
      "External id": 2481111,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567427.422, "dur": 18.344,
    "args": {
      "External id": 2481112,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567451.150, "dur": 3.843,
    "args": {
      "External id": 2481113,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567453.468, "dur": 0.722,
    "args": {
      "External id": 2481114,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567458.675, "dur": 50.095,
    "args": {
      "External id": 2481115,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567459.573, "dur": 5.483,
    "args": {
      "External id": 2481116,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567460.445, "dur": 4.067,
    "args": {
      "External id": 2481117,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567463.766, "dur": 0.599,
    "args": {
      "External id": 2481118,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567465.740, "dur": 42.664,
    "args": {
      "External id": 2481119,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567466.524, "dur": 41.281,
    "args": {
      "External id": 2481120,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567512.936, "dur": 5.397,
    "args": {
      "External id": 2481121,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567515.280, "dur": 2.030,
    "args": {
      "External id": 2481122,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567523.814, "dur": 1.523,
    "args": {
      "External id": 2481123,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567550.203, "dur": 8.512,
    "args": {
      "External id": 2481124,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567553.977, "dur": 4.297,
    "args": {
      "External id": 2481125,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567652.094, "dur": 316.951,
    "args": {
      "External id": 2481126,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567654.282, "dur": 2.397,
    "args": {
      "External id": 2481127,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567658.673, "dur": 309.887,
    "args": {
      "External id": 2481128,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567660.213, "dur": 0.292,
    "args": {
      "External id": 2481129,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567661.682, "dur": 23.103,
    "args": {
      "External id": 2481130,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567686.281, "dur": 3.052,
    "args": {
      "External id": 2481131,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567688.229, "dur": 0.821,
    "args": {
      "External id": 2481132,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567690.171, "dur": 30.000,
    "args": {
      "External id": 2481133,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567692.758, "dur": 1.721,
    "args": {
      "External id": 2481134,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567695.747, "dur": 24.005,
    "args": {
      "External id": 2481135,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567699.985, "dur": 2.743,
    "args": {
      "External id": 2481136,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567721.606, "dur": 36.200,
    "args": {
      "External id": 2481137,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567759.708, "dur": 41.829,
    "args": {
      "External id": 2481138,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567804.221, "dur": 40.336,
    "args": {
      "External id": 2481139,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567845.937, "dur": 42.194,
    "args": {
      "External id": 2481140,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567889.768, "dur": 30.564,
    "args": {
      "External id": 2481141,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567891.912, "dur": 1.580,
    "args": {
      "External id": 2481142,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567897.399, "dur": 0.994,
    "args": {
      "External id": 2481143,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567921.859, "dur": 27.020,
    "args": {
      "External id": 2481144,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567949.848, "dur": 17.637,
    "args": {
      "External id": 2481145,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656567975.863, "dur": 2.342,
    "args": {
      "External id": 2481146,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568027.211, "dur": 4.925,
    "args": {
      "External id": 2481147,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568030.161, "dur": 0.781,
    "args": {
      "External id": 2481148,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568106.465, "dur": 57.321,
    "args": {
      "External id": 2481149,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568170.430, "dur": 4.559,
    "args": {
      "External id": 2481150,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568172.886, "dur": 1.039,
    "args": {
      "External id": 2481151,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568176.459, "dur": 24.967,
    "args": {
      "External id": 2481152,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568237.832, "dur": 5.791,
    "args": {
      "External id": 2481153,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568239.747, "dur": 3.215,
    "args": {
      "External id": 2481154,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568241.677, "dur": 1.074,
    "args": {
      "External id": 2481155,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568248.350, "dur": 43.044,
    "args": {
      "External id": 2481156,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568249.471, "dur": 41.235,
    "args": {
      "External id": 2481157,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568296.069, "dur": 15.976,
    "args": {
      "External id": 2481158,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568318.189, "dur": 3.654,
    "args": {
      "External id": 2481159,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568320.384, "dur": 0.637,
    "args": {
      "External id": 2481160,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568325.813, "dur": 50.712,
    "args": {
      "External id": 2481161,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568326.607, "dur": 7.040,
    "args": {
      "External id": 2481162,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568329.194, "dur": 3.912,
    "args": {
      "External id": 2481163,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568332.252, "dur": 0.720,
    "args": {
      "External id": 2481164,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568334.302, "dur": 41.805,
    "args": {
      "External id": 2481165,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568334.846, "dur": 40.723,
    "args": {
      "External id": 2481166,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568380.460, "dur": 3.671,
    "args": {
      "External id": 2481167,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568382.521, "dur": 0.603,
    "args": {
      "External id": 2481168,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568391.637, "dur": 1.880,
    "args": {
      "External id": 2481169,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568401.663, "dur": 6.099,
    "args": {
      "External id": 2481170,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568403.699, "dur": 3.825,
    "args": {
      "External id": 2481171,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568500.284, "dur": 203.539,
    "args": {
      "External id": 2481172,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568502.726, "dur": 2.614,
    "args": {
      "External id": 2481173,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568506.781, "dur": 196.406,
    "args": {
      "External id": 2481174,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568511.996, "dur": 0.326,
    "args": {
      "External id": 2481175,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568513.664, "dur": 39.809,
    "args": {
      "External id": 2481176,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568556.371, "dur": 3.258,
    "args": {
      "External id": 2481177,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568558.429, "dur": 0.872,
    "args": {
      "External id": 2481178,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568560.743, "dur": 24.762,
    "args": {
      "External id": 2481179,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568563.795, "dur": 1.782,
    "args": {
      "External id": 2481180,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568566.996, "dur": 18.136,
    "args": {
      "External id": 2481181,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568570.289, "dur": 2.737,
    "args": {
      "External id": 2481182,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568587.042, "dur": 21.265,
    "args": {
      "External id": 2481183,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568610.014, "dur": 13.172,
    "args": {
      "External id": 2481184,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568628.493, "dur": 13.334,
    "args": {
      "External id": 2481185,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568643.422, "dur": 12.345,
    "args": {
      "External id": 2481186,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568657.587, "dur": 19.455,
    "args": {
      "External id": 2481187,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568659.624, "dur": 1.372,
    "args": {
      "External id": 2481188,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568663.182, "dur": 0.888,
    "args": {
      "External id": 2481189,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568678.583, "dur": 11.896,
    "args": {
      "External id": 2481190,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568691.679, "dur": 10.524,
    "args": {
      "External id": 2481191,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568713.207, "dur": 2.214,
    "args": {
      "External id": 2481192,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568724.027, "dur": 3.241,
    "args": {
      "External id": 2481193,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568726.095, "dur": 0.391,
    "args": {
      "External id": 2481194,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568797.479, "dur": 53.615,
    "args": {
      "External id": 2481195,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568855.921, "dur": 5.814,
    "args": {
      "External id": 2481196,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568858.315, "dur": 2.338,
    "args": {
      "External id": 2481197,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568863.326, "dur": 24.977,
    "args": {
      "External id": 2481198,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568892.914, "dur": 6.818,
    "args": {
      "External id": 2481199,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568896.260, "dur": 2.719,
    "args": {
      "External id": 2481200,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568897.961, "dur": 0.850,
    "args": {
      "External id": 2481201,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568902.259, "dur": 37.960,
    "args": {
      "External id": 2481202,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568903.396, "dur": 36.201,
    "args": {
      "External id": 2481203,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568944.458, "dur": 12.819,
    "args": {
      "External id": 2481204,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568962.662, "dur": 5.376,
    "args": {
      "External id": 2481205,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568966.393, "dur": 0.729,
    "args": {
      "External id": 2481206,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568971.836, "dur": 88.140,
    "args": {
      "External id": 2481207,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568972.620, "dur": 3.056,
    "args": {
      "External id": 2481208,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568973.273, "dur": 1.943,
    "args": {
      "External id": 2481209,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568974.272, "dur": 0.641,
    "args": {
      "External id": 2481210,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568976.295, "dur": 82.969,
    "args": {
      "External id": 2481211,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656568976.958, "dur": 81.395,
    "args": {
      "External id": 2481212,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569065.731, "dur": 4.297,
    "args": {
      "External id": 2481213,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569068.128, "dur": 0.736,
    "args": {
      "External id": 2481214,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569078.036, "dur": 1.833,
    "args": {
      "External id": 2481215,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569087.715, "dur": 6.508,
    "args": {
      "External id": 2481216,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569089.494, "dur": 4.482,
    "args": {
      "External id": 2481217,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569179.637, "dur": 179.455,
    "args": {
      "External id": 2481218,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569181.834, "dur": 4.323,
    "args": {
      "External id": 2481219,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569189.271, "dur": 169.261,
    "args": {
      "External id": 2481220,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569190.740, "dur": 0.367,
    "args": {
      "External id": 2481221,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569192.376, "dur": 20.749,
    "args": {
      "External id": 2481222,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569214.711, "dur": 5.128,
    "args": {
      "External id": 2481223,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569218.897, "dur": 0.683,
    "args": {
      "External id": 2481224,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569220.768, "dur": 21.131,
    "args": {
      "External id": 2481225,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569221.910, "dur": 1.493,
    "args": {
      "External id": 2481226,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569224.886, "dur": 16.662,
    "args": {
      "External id": 2481227,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569227.266, "dur": 2.665,
    "args": {
      "External id": 2481228,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569245.314, "dur": 19.139,
    "args": {
      "External id": 2481229,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569266.176, "dur": 13.440,
    "args": {
      "External id": 2481230,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569282.603, "dur": 13.191,
    "args": {
      "External id": 2481231,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569297.426, "dur": 11.791,
    "args": {
      "External id": 2481232,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569311.010, "dur": 19.396,
    "args": {
      "External id": 2481233,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569314.599, "dur": 1.127,
    "args": {
      "External id": 2481234,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569317.664, "dur": 0.681,
    "args": {
      "External id": 2481235,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569331.884, "dur": 10.971,
    "args": {
      "External id": 2481236,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569345.841, "dur": 11.764,
    "args": {
      "External id": 2481237,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569365.484, "dur": 1.813,
    "args": {
      "External id": 2481238,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569375.410, "dur": 3.300,
    "args": {
      "External id": 2481239,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569377.421, "dur": 0.400,
    "args": {
      "External id": 2481240,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569445.569, "dur": 48.895,
    "args": {
      "External id": 2481241,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569498.913, "dur": 4.240,
    "args": {
      "External id": 2481242,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569501.332, "dur": 0.780,
    "args": {
      "External id": 2481243,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569504.475, "dur": 38.773,
    "args": {
      "External id": 2481244,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569552.096, "dur": 5.967,
    "args": {
      "External id": 2481245,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569553.637, "dur": 3.516,
    "args": {
      "External id": 2481246,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569555.481, "dur": 1.510,
    "args": {
      "External id": 2481247,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569561.011, "dur": 44.091,
    "args": {
      "External id": 2481248,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569562.407, "dur": 42.056,
    "args": {
      "External id": 2481249,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569609.357, "dur": 15.156,
    "args": {
      "External id": 2481250,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569630.350, "dur": 5.984,
    "args": {
      "External id": 2481251,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569634.556, "dur": 0.909,
    "args": {
      "External id": 2481252,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569639.947, "dur": 47.197,
    "args": {
      "External id": 2481253,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 8956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569640.958, "dur": 3.084,
    "args": {
      "External id": 2481254,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 8957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569641.527, "dur": 2.078,
    "args": {
      "External id": 2481255,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 8958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569642.691, "dur": 0.803,
    "args": {
      "External id": 2481256,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 8959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569644.759, "dur": 41.948,
    "args": {
      "External id": 2481257,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569647.012, "dur": 39.089,
    "args": {
      "External id": 2481258,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 8961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569691.273, "dur": 3.734,
    "args": {
      "External id": 2481259,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569693.287, "dur": 0.500,
    "args": {
      "External id": 2481260,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569709.199, "dur": 1.592,
    "args": {
      "External id": 2481261,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 8964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569721.967, "dur": 9.131,
    "args": {
      "External id": 2481262,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 8965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569725.458, "dur": 5.293,
    "args": {
      "External id": 2481263,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569813.944, "dur": 222.020,
    "args": {
      "External id": 2481264,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569816.143, "dur": 2.210,
    "args": {
      "External id": 2481265,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569820.031, "dur": 215.243,
    "args": {
      "External id": 2481266,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 8969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569831.230, "dur": 0.314,
    "args": {
      "External id": 2481267,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569834.358, "dur": 20.243,
    "args": {
      "External id": 2481268,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 8971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569856.259, "dur": 4.432,
    "args": {
      "External id": 2481269,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 8972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569858.040, "dur": 2.372,
    "args": {
      "External id": 2481270,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 8973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569861.602, "dur": 21.091,
    "args": {
      "External id": 2481271,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569864.378, "dur": 1.630,
    "args": {
      "External id": 2481272,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 8975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569867.106, "dur": 15.253,
    "args": {
      "External id": 2481273,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 8976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569869.239, "dur": 2.565,
    "args": {
      "External id": 2481274,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569884.262, "dur": 19.264,
    "args": {
      "External id": 2481275,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569905.077, "dur": 12.046,
    "args": {
      "External id": 2481276,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569919.812, "dur": 12.845,
    "args": {
      "External id": 2481277,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 8980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569934.343, "dur": 12.213,
    "args": {
      "External id": 2481278,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 8981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569948.341, "dur": 20.878,
    "args": {
      "External id": 2481279,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 8982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569950.062, "dur": 1.415,
    "args": {
      "External id": 2481280,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 8983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569955.249, "dur": 2.010,
    "args": {
      "External id": 2481281,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 8984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656569970.561, "dur": 49.221,
    "args": {
      "External id": 2481282,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 8985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570022.027, "dur": 11.633,
    "args": {
      "External id": 2481283,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 8986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570044.403, "dur": 2.510,
    "args": {
      "External id": 2481284,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 8987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570056.444, "dur": 3.638,
    "args": {
      "External id": 2481285,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 8988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570058.690, "dur": 0.618,
    "args": {
      "External id": 2481286,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 8989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570124.816, "dur": 54.528,
    "args": {
      "External id": 2481287,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 8990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570185.666, "dur": 4.414,
    "args": {
      "External id": 2481288,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 8991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570188.338, "dur": 0.662,
    "args": {
      "External id": 2481289,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 8992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570191.430, "dur": 24.470,
    "args": {
      "External id": 2481290,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 8993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570219.936, "dur": 5.000,
    "args": {
      "External id": 2481291,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 8994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570221.221, "dur": 3.074,
    "args": {
      "External id": 2481292,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 8995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570222.654, "dur": 1.436,
    "args": {
      "External id": 2481293,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 8996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570229.165, "dur": 38.356,
    "args": {
      "External id": 2481294,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570230.175, "dur": 36.764,
    "args": {
      "External id": 2481295,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 8998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570271.717, "dur": 13.414,
    "args": {
      "External id": 2481296,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 8999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570289.976, "dur": 4.086,
    "args": {
      "External id": 2481297,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570292.116, "dur": 1.041,
    "args": {
      "External id": 2481298,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570297.659, "dur": 46.630,
    "args": {
      "External id": 2481299,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570298.548, "dur": 4.972,
    "args": {
      "External id": 2481300,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570301.043, "dur": 2.000,
    "args": {
      "External id": 2481301,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570302.250, "dur": 0.693,
    "args": {
      "External id": 2481302,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570304.019, "dur": 39.936,
    "args": {
      "External id": 2481303,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570304.517, "dur": 38.783,
    "args": {
      "External id": 2481304,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570348.046, "dur": 3.751,
    "args": {
      "External id": 2481305,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570349.874, "dur": 0.829,
    "args": {
      "External id": 2481306,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570356.952, "dur": 1.469,
    "args": {
      "External id": 2481307,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570367.149, "dur": 5.950,
    "args": {
      "External id": 2481308,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570368.721, "dur": 4.118,
    "args": {
      "External id": 2481309,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570449.143, "dur": 190.147,
    "args": {
      "External id": 2481310,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570453.076, "dur": 2.033,
    "args": {
      "External id": 2481311,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570456.540, "dur": 182.286,
    "args": {
      "External id": 2481312,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570457.533, "dur": 0.310,
    "args": {
      "External id": 2481313,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570458.924, "dur": 19.391,
    "args": {
      "External id": 2481314,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570481.592, "dur": 4.547,
    "args": {
      "External id": 2481315,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570485.088, "dur": 0.649,
    "args": {
      "External id": 2481316,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570486.960, "dur": 19.154,
    "args": {
      "External id": 2481317,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570487.894, "dur": 1.460,
    "args": {
      "External id": 2481318,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570490.355, "dur": 15.481,
    "args": {
      "External id": 2481319,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570492.559, "dur": 2.464,
    "args": {
      "External id": 2481320,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570507.557, "dur": 17.459,
    "args": {
      "External id": 2481321,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570542.668, "dur": 18.356,
    "args": {
      "External id": 2481322,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570564.531, "dur": 13.165,
    "args": {
      "External id": 2481323,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570579.171, "dur": 11.238,
    "args": {
      "External id": 2481324,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570593.962, "dur": 19.284,
    "args": {
      "External id": 2481325,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570595.876, "dur": 1.494,
    "args": {
      "External id": 2481326,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570599.444, "dur": 1.011,
    "args": {
      "External id": 2481327,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570614.631, "dur": 11.458,
    "args": {
      "External id": 2481328,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570626.987, "dur": 10.627,
    "args": {
      "External id": 2481329,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570646.739, "dur": 2.155,
    "args": {
      "External id": 2481330,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570656.803, "dur": 3.111,
    "args": {
      "External id": 2481331,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570658.905, "dur": 0.305,
    "args": {
      "External id": 2481332,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570723.255, "dur": 49.977,
    "args": {
      "External id": 2481333,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570777.538, "dur": 4.238,
    "args": {
      "External id": 2481334,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570780.004, "dur": 0.798,
    "args": {
      "External id": 2481335,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570783.085, "dur": 22.904,
    "args": {
      "External id": 2481336,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570810.473, "dur": 6.996,
    "args": {
      "External id": 2481337,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570811.676, "dur": 5.229,
    "args": {
      "External id": 2481338,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570815.447, "dur": 1.235,
    "args": {
      "External id": 2481339,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570819.700, "dur": 37.759,
    "args": {
      "External id": 2481340,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570820.799, "dur": 36.024,
    "args": {
      "External id": 2481341,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570861.302, "dur": 13.013,
    "args": {
      "External id": 2481342,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570879.362, "dur": 3.455,
    "args": {
      "External id": 2481343,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570881.263, "dur": 0.783,
    "args": {
      "External id": 2481344,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570886.299, "dur": 48.711,
    "args": {
      "External id": 2481345,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570888.826, "dur": 2.794,
    "args": {
      "External id": 2481346,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570889.396, "dur": 1.772,
    "args": {
      "External id": 2481347,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570890.521, "dur": 0.509,
    "args": {
      "External id": 2481348,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570892.383, "dur": 42.256,
    "args": {
      "External id": 2481349,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570892.991, "dur": 41.084,
    "args": {
      "External id": 2481350,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570938.945, "dur": 3.594,
    "args": {
      "External id": 2481351,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570940.638, "dur": 0.918,
    "args": {
      "External id": 2481352,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570948.792, "dur": 1.339,
    "args": {
      "External id": 2481353,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570957.049, "dur": 7.036,
    "args": {
      "External id": 2481354,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656570958.631, "dur": 5.181,
    "args": {
      "External id": 2481355,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571083.269, "dur": 180.846,
    "args": {
      "External id": 2481356,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571085.742, "dur": 3.068,
    "args": {
      "External id": 2481357,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571091.806, "dur": 171.705,
    "args": {
      "External id": 2481358,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571093.274, "dur": 0.346,
    "args": {
      "External id": 2481359,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571094.723, "dur": 23.500,
    "args": {
      "External id": 2481360,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571119.843, "dur": 3.440,
    "args": {
      "External id": 2481361,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571122.000, "dur": 0.978,
    "args": {
      "External id": 2481362,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571124.227, "dur": 26.814,
    "args": {
      "External id": 2481363,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571125.229, "dur": 1.511,
    "args": {
      "External id": 2481364,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571128.049, "dur": 22.608,
    "args": {
      "External id": 2481365,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571132.929, "dur": 2.608,
    "args": {
      "External id": 2481366,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571154.028, "dur": 20.305,
    "args": {
      "External id": 2481367,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571175.851, "dur": 11.927,
    "args": {
      "External id": 2481368,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571190.283, "dur": 13.788,
    "args": {
      "External id": 2481369,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571205.602, "dur": 11.136,
    "args": {
      "External id": 2481370,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571218.257, "dur": 18.282,
    "args": {
      "External id": 2481371,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571219.999, "dur": 1.364,
    "args": {
      "External id": 2481372,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571223.392, "dur": 0.725,
    "args": {
      "External id": 2481373,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571237.952, "dur": 11.626,
    "args": {
      "External id": 2481374,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571252.573, "dur": 9.731,
    "args": {
      "External id": 2481375,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571270.896, "dur": 2.068,
    "args": {
      "External id": 2481376,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571281.212, "dur": 3.367,
    "args": {
      "External id": 2481377,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571283.358, "dur": 0.465,
    "args": {
      "External id": 2481378,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571345.477, "dur": 57.021,
    "args": {
      "External id": 2481379,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571406.989, "dur": 4.840,
    "args": {
      "External id": 2481380,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571409.598, "dur": 1.128,
    "args": {
      "External id": 2481381,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571413.264, "dur": 23.182,
    "args": {
      "External id": 2481382,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571442.554, "dur": 5.213,
    "args": {
      "External id": 2481383,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571443.998, "dur": 3.132,
    "args": {
      "External id": 2481384,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571445.810, "dur": 1.091,
    "args": {
      "External id": 2481385,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571450.514, "dur": 39.022,
    "args": {
      "External id": 2481386,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571451.732, "dur": 37.266,
    "args": {
      "External id": 2481387,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571493.724, "dur": 13.568,
    "args": {
      "External id": 2481388,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571511.827, "dur": 46.359,
    "args": {
      "External id": 2481389,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571516.036, "dur": 41.444,
    "args": {
      "External id": 2481390,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571522.983, "dur": 0.586,
    "args": {
      "External id": 2481391,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571566.387, "dur": 28.556,
    "args": {
      "External id": 2481392,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571568.095, "dur": 26.630,
    "args": {
      "External id": 2481393,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 9096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571573.138, "dur": 4.495,
    "args": {
      "External id": 2481394,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571578.833, "dur": 15.391,
    "args": {
      "External id": 2481395,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571610.036, "dur": 6.396,
    "args": {
      "External id": 2481396,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571612.342, "dur": 3.828,
    "args": {
      "External id": 2481397,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571617.424, "dur": 2.928,
    "args": {
      "External id": 2481398,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571619.782, "dur": 0.481,
    "args": {
      "External id": 2481399,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571662.325, "dur": 21.680,
    "args": {
      "External id": 2481400,"Sequence number": 24740625, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9103
    }
  },
  {
    "ph": "s", "id": 23, "pid": 1336757, "tid": 1336757, "ts": 1514656571662.325,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571690.995, "dur": 10.805,
    "args": {
      "External id": 2481401,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 9104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571699.429, "dur": 1.104,
    "args": {
      "External id": 2481402,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571704.253, "dur": 6.221,
    "args": {
      "External id": 2481403,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "1"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 9106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571708.204, "dur": 0.968,
    "args": {
      "External id": 2481404,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "2"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571713.268, "dur": 2.885,
    "args": {
      "External id": 2481405,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 9108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571714.829, "dur": 0.750,
    "args": {
      "External id": 2481406,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "2"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 9109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571720.086, "dur": 5.517,
    "args": {
      "External id": 2481407,"Sequence number": 24740626, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9110
    }
  },
  {
    "ph": "s", "id": 22, "pid": 1336757, "tid": 1336757, "ts": 1514656571720.086,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571723.188, "dur": 1.083,
    "args": {
      "External id": 2481408,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571726.633, "dur": 6.007,
    "args": {
      "External id": 2481409,"Sequence number": 24740627, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9112
    }
  },
  {
    "ph": "s", "id": 21, "pid": 1336757, "tid": 1336757, "ts": 1514656571726.633,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571730.959, "dur": 0.901,
    "args": {
      "External id": 2481410,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571733.513, "dur": 4.774,
    "args": {
      "External id": 2481411,"Sequence number": 24740628, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 9114
    }
  },
  {
    "ph": "s", "id": 20, "pid": 1336757, "tid": 1336757, "ts": 1514656571733.513,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571736.786, "dur": 0.708,
    "args": {
      "External id": 2481412,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571739.187, "dur": 4.479,
    "args": {
      "External id": 2481413,"Sequence number": 24740629, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 9116
    }
  },
  {
    "ph": "s", "id": 19, "pid": 1336757, "tid": 1336757, "ts": 1514656571739.187,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571741.792, "dur": 1.108,
    "args": {
      "External id": 2481414,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "2048"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 9117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571747.696, "dur": 29.468,
    "args": {
      "External id": 2481415,"Sequence number": 24740630, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571749.092, "dur": 27.851,
    "args": {
      "External id": 2481416,"Sequence number": 24740630, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571751.352, "dur": 8.012,
    "args": {
      "External id": 2481417,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 9120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571755.372, "dur": 3.367,
    "args": {
      "External id": 2481418,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571760.273, "dur": 16.190,
    "args": {
      "External id": 2481419,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 9122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571803.210, "dur": 3.693,
    "args": {
      "External id": 2481420,"Sequence number": 24740630, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 9123
    }
  },
  {
    "ph": "s", "id": 18, "pid": 1336757, "tid": 1336757, "ts": 1514656571803.210,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571809.408, "dur": 1.272,
    "args": {
      "External id": 2481421,"Sequence number": 24740631, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571842.763, "dur": 41868.870,
    "args": {
      "External id": 2481422,"Sequence number": 24740631, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 9125
    }
  },
  {
    "ph": "s", "id": 17, "pid": 1336757, "tid": 1336757, "ts": 1514656571842.763,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571857.556, "dur": 26.523,
    "args": {
      "External id": 2481423,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571858.347, "dur": 25.510,
    "args": {
      "External id": 2481424,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571859.775, "dur": 8.122,
    "args": {
      "External id": 2481425,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571864.510, "dur": 3.048,
    "args": {
      "External id": 2481426,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571868.653, "dur": 14.671,
    "args": {
      "External id": 2481427,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 9130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571899.927, "dur": 24.325,
    "args": {
      "External id": 2481428,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571900.825, "dur": 5.446,
    "args": {
      "External id": 2481429,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571902.805, "dur": 3.186,
    "args": {
      "External id": 2481430,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571907.405, "dur": 16.621,
    "args": {
      "External id": 2481431,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571909.033, "dur": 14.601,
    "args": {
      "External id": 2481432,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571927.918, "dur": 19.129,
    "args": {
      "External id": 2481433,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571928.711, "dur": 3.742,
    "args": {
      "External id": 2481434,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571929.666, "dur": 2.544,
    "args": {
      "External id": 2481435,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571932.990, "dur": 13.869,
    "args": {
      "External id": 2481436,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571935.113, "dur": 11.311,
    "args": {
      "External id": 2481437,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 9140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571953.244, "dur": 15.518,
    "args": {
      "External id": 2481438,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 9141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571954.469, "dur": 2.773,
    "args": {
      "External id": 2481439,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571957.857, "dur": 10.635,
    "args": {
      "External id": 2481440,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 9143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571958.665, "dur": 9.518,
    "args": {
      "External id": 2481441,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336757, "tid": 1336757,
    "ts": 1514656571973.949, "dur": 58.426,
    "args": {
      "External id": 2481442,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656572036.708, "dur": 69.655,
    "args": {
      "External id": 2481443,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656572054.865, "dur": 51.038,
    "args": {
      "External id": 2481444,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656572061.814, "dur": 1.006,
    "args": {
      "External id": 2481445,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656572064.017, "dur": 26.912,
    "args": {
      "External id": 2481446,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656572067.313, "dur": 23.363,
    "args": {
      "External id": 2481447,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 9150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656572069.843, "dur": 3.637,
    "args": {
      "External id": 2481448,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656572074.257, "dur": 15.985,
    "args": {
      "External id": 2481449,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 9152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336757, "tid": 1336757,
    "ts": 1514656572110.947, "dur": 36167.737,
    "args": {
      "External id": 2481450,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336757, "tid": 1336757,
    "ts": 1514656572112.335, "dur": 36165.614,
    "args": {
      "External id": 2481451,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608288.320, "dur": 6.274,
    "args": {
      "External id": 2481452,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608291.940, "dur": 0.881,
    "args": {
      "External id": 2481453,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608299.600, "dur": 97.718,
    "args": {
      "External id": 2481454,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608302.953, "dur": 5.957,
    "args": {
      "External id": 2481455,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608305.214, "dur": 2.985,
    "args": {
      "External id": 2481456,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608306.858, "dur": 1.041,
    "args": {
      "External id": 2481457,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608310.144, "dur": 86.462,
    "args": {
      "External id": 2481458,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608311.525, "dur": 84.386,
    "args": {
      "External id": 2481459,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608400.941, "dur": 4.112,
    "args": {
      "External id": 2481460,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608402.969, "dur": 0.868,
    "args": {
      "External id": 2481461,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608413.327, "dur": 2.110,
    "args": {
      "External id": 2481462,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608423.249, "dur": 5.683,
    "args": {
      "External id": 2481463,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608424.935, "dur": 3.671,
    "args": {
      "External id": 2481464,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608560.008, "dur": 192.465,
    "args": {
      "External id": 2481465,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608563.032, "dur": 3.671,
    "args": {
      "External id": 2481466,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608568.454, "dur": 183.495,
    "args": {
      "External id": 2481467,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608572.259, "dur": 0.374,
    "args": {
      "External id": 2481468,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608574.093, "dur": 27.141,
    "args": {
      "External id": 2481469,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608602.860, "dur": 3.414,
    "args": {
      "External id": 2481470,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608605.011, "dur": 1.012,
    "args": {
      "External id": 2481471,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608607.229, "dur": 23.357,
    "args": {
      "External id": 2481472,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608608.257, "dur": 1.554,
    "args": {
      "External id": 2481473,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608611.032, "dur": 19.256,
    "args": {
      "External id": 2481474,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608615.703, "dur": 3.328,
    "args": {
      "External id": 2481475,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608632.230, "dur": 21.396,
    "args": {
      "External id": 2481476,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608655.480, "dur": 13.377,
    "args": {
      "External id": 2481477,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608673.379, "dur": 14.154,
    "args": {
      "External id": 2481478,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608689.234, "dur": 12.366,
    "args": {
      "External id": 2481479,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608703.043, "dur": 20.363,
    "args": {
      "External id": 2481480,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608705.404, "dur": 1.567,
    "args": {
      "External id": 2481481,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608709.138, "dur": 0.909,
    "args": {
      "External id": 2481482,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608725.178, "dur": 12.294,
    "args": {
      "External id": 2481483,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608740.711, "dur": 9.991,
    "args": {
      "External id": 2481484,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608759.895, "dur": 2.267,
    "args": {
      "External id": 2481485,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608767.969, "dur": 3.913,
    "args": {
      "External id": 2481486,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608770.161, "dur": 0.773,
    "args": {
      "External id": 2481487,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608840.977, "dur": 63.271,
    "args": {
      "External id": 2481488,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608910.548, "dur": 5.725,
    "args": {
      "External id": 2481489,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608913.046, "dur": 0.883,
    "args": {
      "External id": 2481490,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608917.886, "dur": 29.630,
    "args": {
      "External id": 2481491,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608954.378, "dur": 5.517,
    "args": {
      "External id": 2481492,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608956.220, "dur": 2.876,
    "args": {
      "External id": 2481493,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608957.967, "dur": 0.871,
    "args": {
      "External id": 2481494,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608962.718, "dur": 78.478,
    "args": {
      "External id": 2481495,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656608963.885, "dur": 76.084,
    "args": {
      "External id": 2481496,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609047.534, "dur": 16.070,
    "args": {
      "External id": 2481497,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609070.536, "dur": 6.687,
    "args": {
      "External id": 2481498,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609074.997, "dur": 0.984,
    "args": {
      "External id": 2481499,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609081.969, "dur": 61.446,
    "args": {
      "External id": 2481500,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609083.157, "dur": 5.610,
    "args": {
      "External id": 2481501,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609083.993, "dur": 4.217,
    "args": {
      "External id": 2481502,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609085.474, "dur": 2.536,
    "args": {
      "External id": 2481503,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609089.517, "dur": 53.438,
    "args": {
      "External id": 2481504,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609096.850, "dur": 45.573,
    "args": {
      "External id": 2481505,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609147.844, "dur": 3.804,
    "args": {
      "External id": 2481506,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609149.984, "dur": 0.625,
    "args": {
      "External id": 2481507,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609157.783, "dur": 1.694,
    "args": {
      "External id": 2481508,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609166.410, "dur": 8.073,
    "args": {
      "External id": 2481509,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609170.028, "dur": 4.161,
    "args": {
      "External id": 2481510,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609261.575, "dur": 182.567,
    "args": {
      "External id": 2481511,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609265.743, "dur": 2.091,
    "args": {
      "External id": 2481512,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609269.226, "dur": 174.400,
    "args": {
      "External id": 2481513,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609270.667, "dur": 0.296,
    "args": {
      "External id": 2481514,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609272.154, "dur": 21.404,
    "args": {
      "External id": 2481515,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609296.868, "dur": 8.503,
    "args": {
      "External id": 2481516,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609304.389, "dur": 0.725,
    "args": {
      "External id": 2481517,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609308.121, "dur": 20.224,
    "args": {
      "External id": 2481518,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609309.138, "dur": 1.365,
    "args": {
      "External id": 2481519,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609311.782, "dur": 16.156,
    "args": {
      "External id": 2481520,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609314.076, "dur": 2.913,
    "args": {
      "External id": 2481521,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609330.142, "dur": 20.998,
    "args": {
      "External id": 2481522,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609352.563, "dur": 14.790,
    "args": {
      "External id": 2481523,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609370.001, "dur": 13.641,
    "args": {
      "External id": 2481524,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609385.252, "dur": 11.641,
    "args": {
      "External id": 2481525,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609398.332, "dur": 20.142,
    "args": {
      "External id": 2481526,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609401.828, "dur": 1.337,
    "args": {
      "External id": 2481527,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609405.212, "dur": 0.703,
    "args": {
      "External id": 2481528,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609420.002, "dur": 11.088,
    "args": {
      "External id": 2481529,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609432.364, "dur": 10.229,
    "args": {
      "External id": 2481530,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609450.449, "dur": 1.594,
    "args": {
      "External id": 2481531,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609459.575, "dur": 3.025,
    "args": {
      "External id": 2481532,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609461.484, "dur": 0.351,
    "args": {
      "External id": 2481533,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609523.986, "dur": 74.813,
    "args": {
      "External id": 2481534,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609606.819, "dur": 5.675,
    "args": {
      "External id": 2481535,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609610.096, "dur": 0.976,
    "args": {
      "External id": 2481536,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609613.928, "dur": 25.867,
    "args": {
      "External id": 2481537,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609644.987, "dur": 5.342,
    "args": {
      "External id": 2481538,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609646.125, "dur": 3.478,
    "args": {
      "External id": 2481539,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609647.969, "dur": 1.398,
    "args": {
      "External id": 2481540,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609654.963, "dur": 39.590,
    "args": {
      "External id": 2481541,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609655.885, "dur": 38.063,
    "args": {
      "External id": 2481542,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609699.145, "dur": 15.964,
    "args": {
      "External id": 2481543,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609720.815, "dur": 3.744,
    "args": {
      "External id": 2481544,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609722.903, "dur": 0.770,
    "args": {
      "External id": 2481545,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609728.351, "dur": 47.367,
    "args": {
      "External id": 2481546,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609729.235, "dur": 5.206,
    "args": {
      "External id": 2481547,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609731.937, "dur": 1.972,
    "args": {
      "External id": 2481548,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609733.061, "dur": 0.701,
    "args": {
      "External id": 2481549,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609735.152, "dur": 40.175,
    "args": {
      "External id": 2481550,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609735.695, "dur": 39.063,
    "args": {
      "External id": 2481551,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609779.813, "dur": 4.313,
    "args": {
      "External id": 2481552,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609782.092, "dur": 0.944,
    "args": {
      "External id": 2481553,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609790.319, "dur": 1.375,
    "args": {
      "External id": 2481554,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609800.590, "dur": 7.622,
    "args": {
      "External id": 2481555,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609802.451, "dur": 5.462,
    "args": {
      "External id": 2481556,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609896.833, "dur": 217.055,
    "args": {
      "External id": 2481557,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609898.854, "dur": 2.247,
    "args": {
      "External id": 2481558,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609902.334, "dur": 211.006,
    "args": {
      "External id": 2481559,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609903.840, "dur": 0.310,
    "args": {
      "External id": 2481560,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609905.203, "dur": 22.085,
    "args": {
      "External id": 2481561,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609928.648, "dur": 3.174,
    "args": {
      "External id": 2481562,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609930.753, "dur": 0.675,
    "args": {
      "External id": 2481563,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609934.577, "dur": 21.395,
    "args": {
      "External id": 2481564,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609935.769, "dur": 1.472,
    "args": {
      "External id": 2481565,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609938.280, "dur": 17.386,
    "args": {
      "External id": 2481566,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609942.619, "dur": 2.395,
    "args": {
      "External id": 2481567,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609957.554, "dur": 21.887,
    "args": {
      "External id": 2481568,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656609980.794, "dur": 50.460,
    "args": {
      "External id": 2481569,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610035.682, "dur": 14.131,
    "args": {
      "External id": 2481570,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610051.289, "dur": 12.130,
    "args": {
      "External id": 2481571,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610065.230, "dur": 22.319,
    "args": {
      "External id": 2481572,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610069.229, "dur": 2.003,
    "args": {
      "External id": 2481573,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610073.498, "dur": 1.013,
    "args": {
      "External id": 2481574,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610089.059, "dur": 11.688,
    "args": {
      "External id": 2481575,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610101.884, "dur": 10.282,
    "args": {
      "External id": 2481576,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610122.145, "dur": 2.416,
    "args": {
      "External id": 2481577,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610134.360, "dur": 3.469,
    "args": {
      "External id": 2481578,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610136.683, "dur": 0.387,
    "args": {
      "External id": 2481579,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610209.948, "dur": 75.795,
    "args": {
      "External id": 2481580,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610290.948, "dur": 4.753,
    "args": {
      "External id": 2481581,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610293.497, "dur": 1.072,
    "args": {
      "External id": 2481582,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610297.148, "dur": 30.585,
    "args": {
      "External id": 2481583,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610332.429, "dur": 6.668,
    "args": {
      "External id": 2481584,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610333.927, "dur": 4.446,
    "args": {
      "External id": 2481585,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610337.083, "dur": 1.055,
    "args": {
      "External id": 2481586,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610341.825, "dur": 70.770,
    "args": {
      "External id": 2481587,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610342.880, "dur": 69.073,
    "args": {
      "External id": 2481588,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610416.620, "dur": 37.723,
    "args": {
      "External id": 2481589,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610459.500, "dur": 3.922,
    "args": {
      "External id": 2481590,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610461.805, "dur": 0.713,
    "args": {
      "External id": 2481591,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610467.401, "dur": 49.969,
    "args": {
      "External id": 2481592,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610469.846, "dur": 4.906,
    "args": {
      "External id": 2481593,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610470.571, "dur": 3.576,
    "args": {
      "External id": 2481594,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610471.761, "dur": 2.208,
    "args": {
      "External id": 2481595,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610475.567, "dur": 41.357,
    "args": {
      "External id": 2481596,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610476.105, "dur": 40.176,
    "args": {
      "External id": 2481597,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610521.075, "dur": 4.092,
    "args": {
      "External id": 2481598,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610523.282, "dur": 0.703,
    "args": {
      "External id": 2481599,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610547.963, "dur": 2.212,
    "args": {
      "External id": 2481600,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610558.278, "dur": 6.434,
    "args": {
      "External id": 2481601,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610559.886, "dur": 4.575,
    "args": {
      "External id": 2481602,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610649.524, "dur": 171.113,
    "args": {
      "External id": 2481603,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610652.123, "dur": 2.188,
    "args": {
      "External id": 2481604,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610657.865, "dur": 162.373,
    "args": {
      "External id": 2481605,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610659.138, "dur": 0.259,
    "args": {
      "External id": 2481606,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610662.223, "dur": 21.191,
    "args": {
      "External id": 2481607,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610684.788, "dur": 3.238,
    "args": {
      "External id": 2481608,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610686.814, "dur": 0.849,
    "args": {
      "External id": 2481609,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610688.835, "dur": 20.239,
    "args": {
      "External id": 2481610,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610689.815, "dur": 2.753,
    "args": {
      "External id": 2481611,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610693.678, "dur": 15.074,
    "args": {
      "External id": 2481612,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610696.121, "dur": 2.204,
    "args": {
      "External id": 2481613,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610711.847, "dur": 20.546,
    "args": {
      "External id": 2481614,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610733.966, "dur": 11.830,
    "args": {
      "External id": 2481615,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610748.740, "dur": 12.737,
    "args": {
      "External id": 2481616,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610762.913, "dur": 11.329,
    "args": {
      "External id": 2481617,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610775.975, "dur": 18.356,
    "args": {
      "External id": 2481618,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610777.863, "dur": 1.128,
    "args": {
      "External id": 2481619,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610781.145, "dur": 0.885,
    "args": {
      "External id": 2481620,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610795.806, "dur": 10.615,
    "args": {
      "External id": 2481621,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610809.292, "dur": 9.857,
    "args": {
      "External id": 2481622,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610826.666, "dur": 1.505,
    "args": {
      "External id": 2481623,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610836.526, "dur": 3.582,
    "args": {
      "External id": 2481624,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610838.946, "dur": 0.393,
    "args": {
      "External id": 2481625,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610900.330, "dur": 50.353,
    "args": {
      "External id": 2481626,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610955.530, "dur": 4.642,
    "args": {
      "External id": 2481627,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610958.169, "dur": 0.819,
    "args": {
      "External id": 2481628,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656610961.576, "dur": 59.547,
    "args": {
      "External id": 2481629,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611029.318, "dur": 6.949,
    "args": {
      "External id": 2481630,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611030.759, "dur": 4.698,
    "args": {
      "External id": 2481631,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611032.562, "dur": 2.673,
    "args": {
      "External id": 2481632,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611039.103, "dur": 44.238,
    "args": {
      "External id": 2481633,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611040.115, "dur": 42.525,
    "args": {
      "External id": 2481634,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611087.773, "dur": 14.888,
    "args": {
      "External id": 2481635,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611108.215, "dur": 6.456,
    "args": {
      "External id": 2481636,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611112.499, "dur": 1.212,
    "args": {
      "External id": 2481637,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611118.562, "dur": 47.427,
    "args": {
      "External id": 2481638,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611119.292, "dur": 3.242,
    "args": {
      "External id": 2481639,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611120.007, "dur": 1.965,
    "args": {
      "External id": 2481640,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611121.077, "dur": 0.750,
    "args": {
      "External id": 2481641,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611123.134, "dur": 42.490,
    "args": {
      "External id": 2481642,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611125.494, "dur": 39.381,
    "args": {
      "External id": 2481643,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611170.233, "dur": 3.642,
    "args": {
      "External id": 2481644,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611172.127, "dur": 0.653,
    "args": {
      "External id": 2481645,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611179.927, "dur": 1.556,
    "args": {
      "External id": 2481646,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611188.533, "dur": 6.327,
    "args": {
      "External id": 2481647,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611190.357, "dur": 4.227,
    "args": {
      "External id": 2481648,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611275.309, "dur": 204.253,
    "args": {
      "External id": 2481649,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611277.349, "dur": 2.341,
    "args": {
      "External id": 2481650,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611282.910, "dur": 196.042,
    "args": {
      "External id": 2481651,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611284.253, "dur": 0.239,
    "args": {
      "External id": 2481652,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611285.610, "dur": 19.059,
    "args": {
      "External id": 2481653,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611306.457, "dur": 4.221,
    "args": {
      "External id": 2481654,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611308.369, "dur": 1.954,
    "args": {
      "External id": 2481655,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611311.643, "dur": 20.894,
    "args": {
      "External id": 2481656,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611312.763, "dur": 1.267,
    "args": {
      "External id": 2481657,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611315.184, "dur": 17.053,
    "args": {
      "External id": 2481658,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611318.972, "dur": 2.573,
    "args": {
      "External id": 2481659,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611333.917, "dur": 18.206,
    "args": {
      "External id": 2481660,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611353.517, "dur": 12.066,
    "args": {
      "External id": 2481661,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611368.195, "dur": 12.973,
    "args": {
      "External id": 2481662,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611382.547, "dur": 11.284,
    "args": {
      "External id": 2481663,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611395.242, "dur": 57.724,
    "args": {
      "External id": 2481664,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611429.958, "dur": 6.035,
    "args": {
      "External id": 2481665,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611438.237, "dur": 2.070,
    "args": {
      "External id": 2481666,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611456.082, "dur": 10.690,
    "args": {
      "External id": 2481667,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611467.722, "dur": 10.285,
    "args": {
      "External id": 2481668,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611486.524, "dur": 1.648,
    "args": {
      "External id": 2481669,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611495.502, "dur": 3.044,
    "args": {
      "External id": 2481670,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611497.518, "dur": 0.257,
    "args": {
      "External id": 2481671,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611581.143, "dur": 54.154,
    "args": {
      "External id": 2481672,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611640.008, "dur": 7.420,
    "args": {
      "External id": 2481673,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611644.752, "dur": 1.177,
    "args": {
      "External id": 2481674,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611648.587, "dur": 23.615,
    "args": {
      "External id": 2481675,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611676.756, "dur": 4.742,
    "args": {
      "External id": 2481676,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611677.894, "dur": 3.029,
    "args": {
      "External id": 2481677,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611679.233, "dur": 1.463,
    "args": {
      "External id": 2481678,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611684.163, "dur": 38.181,
    "args": {
      "External id": 2481679,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611685.035, "dur": 36.640,
    "args": {
      "External id": 2481680,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611727.914, "dur": 14.246,
    "args": {
      "External id": 2481681,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611747.762, "dur": 3.327,
    "args": {
      "External id": 2481682,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611749.676, "dur": 0.587,
    "args": {
      "External id": 2481683,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611754.643, "dur": 44.719,
    "args": {
      "External id": 2481684,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611755.486, "dur": 4.763,
    "args": {
      "External id": 2481685,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611756.267, "dur": 3.511,
    "args": {
      "External id": 2481686,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611758.943, "dur": 0.711,
    "args": {
      "External id": 2481687,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611760.809, "dur": 38.117,
    "args": {
      "External id": 2481688,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611761.218, "dur": 37.083,
    "args": {
      "External id": 2481689,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611802.802, "dur": 3.712,
    "args": {
      "External id": 2481690,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611804.769, "dur": 0.618,
    "args": {
      "External id": 2481691,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611811.915, "dur": 1.393,
    "args": {
      "External id": 2481692,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611821.916, "dur": 5.502,
    "args": {
      "External id": 2481693,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611823.671, "dur": 3.492,
    "args": {
      "External id": 2481694,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611901.433, "dur": 213.034,
    "args": {
      "External id": 2481695,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611905.092, "dur": 2.237,
    "args": {
      "External id": 2481696,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611908.786, "dur": 204.996,
    "args": {
      "External id": 2481697,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611909.936, "dur": 0.285,
    "args": {
      "External id": 2481698,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611911.039, "dur": 18.602,
    "args": {
      "External id": 2481699,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611930.973, "dur": 4.598,
    "args": {
      "External id": 2481700,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611934.046, "dur": 1.207,
    "args": {
      "External id": 2481701,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611938.430, "dur": 21.471,
    "args": {
      "External id": 2481702,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611939.499, "dur": 1.529,
    "args": {
      "External id": 2481703,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611941.901, "dur": 17.710,
    "args": {
      "External id": 2481704,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611944.143, "dur": 2.275,
    "args": {
      "External id": 2481705,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656611961.395, "dur": 55.445,
    "args": {
      "External id": 2481706,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612019.404, "dur": 13.627,
    "args": {
      "External id": 2481707,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612036.148, "dur": 13.900,
    "args": {
      "External id": 2481708,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612051.561, "dur": 11.849,
    "args": {
      "External id": 2481709,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612065.205, "dur": 23.077,
    "args": {
      "External id": 2481710,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612070.291, "dur": 1.691,
    "args": {
      "External id": 2481711,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612074.201, "dur": 1.089,
    "args": {
      "External id": 2481712,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612089.765, "dur": 11.142,
    "args": {
      "External id": 2481713,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612101.994, "dur": 10.557,
    "args": {
      "External id": 2481714,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612122.369, "dur": 2.369,
    "args": {
      "External id": 2481715,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612133.470, "dur": 3.407,
    "args": {
      "External id": 2481716,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612135.705, "dur": 0.355,
    "args": {
      "External id": 2481717,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612202.208, "dur": 51.661,
    "args": {
      "External id": 2481718,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612258.554, "dur": 4.314,
    "args": {
      "External id": 2481719,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612260.866, "dur": 0.842,
    "args": {
      "External id": 2481720,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612264.413, "dur": 22.842,
    "args": {
      "External id": 2481721,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612291.444, "dur": 7.131,
    "args": {
      "External id": 2481722,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612292.984, "dur": 4.641,
    "args": {
      "External id": 2481723,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612295.970, "dur": 1.443,
    "args": {
      "External id": 2481724,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612301.192, "dur": 37.299,
    "args": {
      "External id": 2481725,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612302.176, "dur": 35.592,
    "args": {
      "External id": 2481726,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612342.349, "dur": 12.953,
    "args": {
      "External id": 2481727,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612360.387, "dur": 3.658,
    "args": {
      "External id": 2481728,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612362.445, "dur": 0.734,
    "args": {
      "External id": 2481729,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612367.386, "dur": 46.191,
    "args": {
      "External id": 2481730,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612369.859, "dur": 3.410,
    "args": {
      "External id": 2481731,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612370.604, "dur": 2.176,
    "args": {
      "External id": 2481732,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612371.796, "dur": 0.850,
    "args": {
      "External id": 2481733,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612373.819, "dur": 39.285,
    "args": {
      "External id": 2481734,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612374.499, "dur": 37.927,
    "args": {
      "External id": 2481735,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612417.390, "dur": 4.061,
    "args": {
      "External id": 2481736,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612419.493, "dur": 0.937,
    "args": {
      "External id": 2481737,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612425.971, "dur": 1.347,
    "args": {
      "External id": 2481738,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612435.858, "dur": 5.347,
    "args": {
      "External id": 2481739,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612437.469, "dur": 3.476,
    "args": {
      "External id": 2481740,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612514.851, "dur": 193.010,
    "args": {
      "External id": 2481741,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612518.411, "dur": 2.217,
    "args": {
      "External id": 2481742,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612522.031, "dur": 185.266,
    "args": {
      "External id": 2481743,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612523.402, "dur": 0.327,
    "args": {
      "External id": 2481744,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612526.185, "dur": 38.175,
    "args": {
      "External id": 2481745,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612567.082, "dur": 4.883,
    "args": {
      "External id": 2481746,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612570.715, "dur": 0.888,
    "args": {
      "External id": 2481747,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612579.573, "dur": 19.970,
    "args": {
      "External id": 2481748,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612580.710, "dur": 1.652,
    "args": {
      "External id": 2481749,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612583.651, "dur": 15.550,
    "args": {
      "External id": 2481750,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612585.956, "dur": 2.792,
    "args": {
      "External id": 2481751,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612600.760, "dur": 20.592,
    "args": {
      "External id": 2481752,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612622.843, "dur": 11.962,
    "args": {
      "External id": 2481753,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612638.908, "dur": 12.388,
    "args": {
      "External id": 2481754,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612652.752, "dur": 11.205,
    "args": {
      "External id": 2481755,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612665.393, "dur": 17.348,
    "args": {
      "External id": 2481756,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612666.990, "dur": 1.137,
    "args": {
      "External id": 2481757,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612670.130, "dur": 0.771,
    "args": {
      "External id": 2481758,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612684.104, "dur": 10.729,
    "args": {
      "External id": 2481759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612695.990, "dur": 10.297,
    "args": {
      "External id": 2481760,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612716.794, "dur": 2.141,
    "args": {
      "External id": 2481761,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612726.890, "dur": 3.057,
    "args": {
      "External id": 2481762,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612728.915, "dur": 0.316,
    "args": {
      "External id": 2481763,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612795.465, "dur": 53.924,
    "args": {
      "External id": 2481764,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612853.865, "dur": 4.515,
    "args": {
      "External id": 2481765,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612856.387, "dur": 0.950,
    "args": {
      "External id": 2481766,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612862.636, "dur": 21.655,
    "args": {
      "External id": 2481767,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612890.264, "dur": 9.446,
    "args": {
      "External id": 2481768,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612891.648, "dur": 7.358,
    "args": {
      "External id": 2481769,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612896.564, "dur": 2.182,
    "args": {
      "External id": 2481770,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612902.166, "dur": 39.369,
    "args": {
      "External id": 2481771,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612903.155, "dur": 37.821,
    "args": {
      "External id": 2481772,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612945.624, "dur": 13.352,
    "args": {
      "External id": 2481773,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612964.264, "dur": 4.893,
    "args": {
      "External id": 2481774,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612967.630, "dur": 0.729,
    "args": {
      "External id": 2481775,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612972.603, "dur": 87.169,
    "args": {
      "External id": 2481776,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612973.516, "dur": 3.596,
    "args": {
      "External id": 2481777,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612974.447, "dur": 2.143,
    "args": {
      "External id": 2481778,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612975.807, "dur": 0.638,
    "args": {
      "External id": 2481779,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612977.764, "dur": 81.455,
    "args": {
      "External id": 2481780,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656612979.829, "dur": 78.327,
    "args": {
      "External id": 2481781,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613065.996, "dur": 4.762,
    "args": {
      "External id": 2481782,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613068.440, "dur": 0.914,
    "args": {
      "External id": 2481783,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613077.269, "dur": 1.703,
    "args": {
      "External id": 2481784,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613087.055, "dur": 7.803,
    "args": {
      "External id": 2481785,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613090.100, "dur": 4.458,
    "args": {
      "External id": 2481786,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613182.966, "dur": 170.092,
    "args": {
      "External id": 2481787,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613184.646, "dur": 2.505,
    "args": {
      "External id": 2481788,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613190.615, "dur": 161.892,
    "args": {
      "External id": 2481789,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613191.790, "dur": 0.347,
    "args": {
      "External id": 2481790,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613193.276, "dur": 20.666,
    "args": {
      "External id": 2481791,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613215.293, "dur": 4.640,
    "args": {
      "External id": 2481792,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613217.409, "dur": 2.213,
    "args": {
      "External id": 2481793,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613220.792, "dur": 21.218,
    "args": {
      "External id": 2481794,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613223.333, "dur": 1.598,
    "args": {
      "External id": 2481795,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613226.032, "dur": 15.583,
    "args": {
      "External id": 2481796,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613228.558, "dur": 2.532,
    "args": {
      "External id": 2481797,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613243.302, "dur": 19.600,
    "args": {
      "External id": 2481798,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613264.352, "dur": 12.192,
    "args": {
      "External id": 2481799,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613279.238, "dur": 12.973,
    "args": {
      "External id": 2481800,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613293.713, "dur": 11.187,
    "args": {
      "External id": 2481801,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613306.314, "dur": 21.462,
    "args": {
      "External id": 2481802,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613308.077, "dur": 1.189,
    "args": {
      "External id": 2481803,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613312.836, "dur": 2.221,
    "args": {
      "External id": 2481804,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613329.158, "dur": 10.781,
    "args": {
      "External id": 2481805,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613340.935, "dur": 10.777,
    "args": {
      "External id": 2481806,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613358.950, "dur": 1.680,
    "args": {
      "External id": 2481807,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613368.510, "dur": 3.100,
    "args": {
      "External id": 2481808,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613370.428, "dur": 0.402,
    "args": {
      "External id": 2481809,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613429.959, "dur": 47.073,
    "args": {
      "External id": 2481810,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613481.211, "dur": 6.242,
    "args": {
      "External id": 2481811,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613485.307, "dur": 0.965,
    "args": {
      "External id": 2481812,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613488.689, "dur": 22.602,
    "args": {
      "External id": 2481813,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613515.218, "dur": 4.998,
    "args": {
      "External id": 2481814,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613516.716, "dur": 2.775,
    "args": {
      "External id": 2481815,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613518.149, "dur": 1.141,
    "args": {
      "External id": 2481816,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613522.701, "dur": 64.808,
    "args": {
      "External id": 2481817,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613525.364, "dur": 61.210,
    "args": {
      "External id": 2481818,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613592.853, "dur": 14.826,
    "args": {
      "External id": 2481819,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613612.362, "dur": 23.480,
    "args": {
      "External id": 2481820,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613614.881, "dur": 20.588,
    "args": {
      "External id": 2481821,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613619.979, "dur": 1.102,
    "args": {
      "External id": 2481822,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613641.554, "dur": 24.466,
    "args": {
      "External id": 2481823,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613643.262, "dur": 22.553,
    "args": {
      "External id": 2481824,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 9527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613647.481, "dur": 3.989,
    "args": {
      "External id": 2481825,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613652.462, "dur": 12.866,
    "args": {
      "External id": 2481826,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613678.949, "dur": 6.650,
    "args": {
      "External id": 2481827,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613682.321, "dur": 2.985,
    "args": {
      "External id": 2481828,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613686.663, "dur": 0.852,
    "args": {
      "External id": 2481829,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613686.988, "dur": 0.445,
    "args": {
      "External id": 2481830,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613726.886, "dur": 21.055,
    "args": {
      "External id": 2481831,"Sequence number": 24740632, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613749.911, "dur": 12.074,
    "args": {
      "External id": 2481832,"Sequence number": 24740633, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9535
    }
  },
  {
    "ph": "s", "id": 16, "pid": 1336757, "tid": 1336757, "ts": 1514656613749.911,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613768.185, "dur": 5.758,
    "args": {
      "External id": 2481833,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 9536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613771.664, "dur": 1.053,
    "args": {
      "External id": 2481834,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613778.257, "dur": 6.191,
    "args": {
      "External id": 2481835,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "2"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 9538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613782.308, "dur": 0.794,
    "args": {
      "External id": 2481836,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "3"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613785.785, "dur": 2.568,
    "args": {
      "External id": 2481837,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 9540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613787.102, "dur": 0.705,
    "args": {
      "External id": 2481838,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "3"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 9541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613792.243, "dur": 6.856,
    "args": {
      "External id": 2481839,"Sequence number": 24740634, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9542
    }
  },
  {
    "ph": "s", "id": 15, "pid": 1336757, "tid": 1336757, "ts": 1514656613792.243,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613796.927, "dur": 0.876,
    "args": {
      "External id": 2481840,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613799.996, "dur": 4.851,
    "args": {
      "External id": 2481841,"Sequence number": 24740635, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9544
    }
  },
  {
    "ph": "s", "id": 14, "pid": 1336757, "tid": 1336757, "ts": 1514656613799.996,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613803.169, "dur": 0.931,
    "args": {
      "External id": 2481842,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613805.843, "dur": 6.712,
    "args": {
      "External id": 2481843,"Sequence number": 24740636, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "2"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 9546
    }
  },
  {
    "ph": "s", "id": 13, "pid": 1336757, "tid": 1336757, "ts": 1514656613805.843,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613810.869, "dur": 0.700,
    "args": {
      "External id": 2481844,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613813.674, "dur": 4.687,
    "args": {
      "External id": 2481845,"Sequence number": 24740637, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 9548
    }
  },
  {
    "ph": "s", "id": 12, "pid": 1336757, "tid": 1336757, "ts": 1514656613813.674,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613816.515, "dur": 0.952,
    "args": {
      "External id": 2481846,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "4096"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 9549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613822.038, "dur": 29.476,
    "args": {
      "External id": 2481847,"Sequence number": 24740638, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613823.591, "dur": 27.694,
    "args": {
      "External id": 2481848,"Sequence number": 24740638, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613826.179, "dur": 5.670,
    "args": {
      "External id": 2481849,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 9552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613828.119, "dur": 3.174,
    "args": {
      "External id": 2481850,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613832.638, "dur": 18.175,
    "args": {
      "External id": 2481851,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 9554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613878.279, "dur": 5.959,
    "args": {
      "External id": 2481852,"Sequence number": 24740638, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 9555
    }
  },
  {
    "ph": "s", "id": 11, "pid": 1336757, "tid": 1336757, "ts": 1514656613878.279,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613886.482, "dur": 1.156,
    "args": {
      "External id": 2481853,"Sequence number": 24740639, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613921.663, "dur": 43487.206,
    "args": {
      "External id": 2481854,"Sequence number": 24740639, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 9557
    }
  },
  {
    "ph": "s", "id": 10, "pid": 1336757, "tid": 1336757, "ts": 1514656613921.663,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613935.769, "dur": 23.957,
    "args": {
      "External id": 2481855,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613936.444, "dur": 23.099,
    "args": {
      "External id": 2481856,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613937.770, "dur": 5.077,
    "args": {
      "External id": 2481857,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613939.046, "dur": 3.447,
    "args": {
      "External id": 2481858,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613943.602, "dur": 15.442,
    "args": {
      "External id": 2481859,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 9562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613974.813, "dur": 63.103,
    "args": {
      "External id": 2481860,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613975.853, "dur": 6.196,
    "args": {
      "External id": 2481861,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613977.735, "dur": 4.031,
    "args": {
      "External id": 2481862,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656613983.338, "dur": 54.352,
    "args": {
      "External id": 2481863,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614020.945, "dur": 16.094,
    "args": {
      "External id": 2481864,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614042.601, "dur": 19.731,
    "args": {
      "External id": 2481865,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614043.685, "dur": 6.911,
    "args": {
      "External id": 2481866,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614046.905, "dur": 3.345,
    "args": {
      "External id": 2481867,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614051.095, "dur": 11.040,
    "args": {
      "External id": 2481868,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614051.627, "dur": 10.195,
    "args": {
      "External id": 2481869,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 9572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614069.038, "dur": 17.541,
    "args": {
      "External id": 2481870,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 9573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614072.317, "dur": 3.235,
    "args": {
      "External id": 2481871,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614076.092, "dur": 10.224,
    "args": {
      "External id": 2481872,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 9575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614076.784, "dur": 9.229,
    "args": {
      "External id": 2481873,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614093.175, "dur": 22.364,
    "args": {
      "External id": 2481874,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614118.114, "dur": 45.316,
    "args": {
      "External id": 2481875,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614120.167, "dur": 42.798,
    "args": {
      "External id": 2481876,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614124.793, "dur": 1.225,
    "args": {
      "External id": 2481877,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614127.173, "dur": 21.294,
    "args": {
      "External id": 2481878,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614128.745, "dur": 19.516,
    "args": {
      "External id": 2481879,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 9582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614132.665, "dur": 2.798,
    "args": {
      "External id": 2481880,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614136.165, "dur": 11.704,
    "args": {
      "External id": 2481881,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 9584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614167.134, "dur": 37776.531,
    "args": {
      "External id": 2481882,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336757, "tid": 1336757,
    "ts": 1514656614168.509, "dur": 37774.222,
    "args": {
      "External id": 2481883,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 9586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656651955.801, "dur": 9.242,
    "args": {
      "External id": 2481884,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656651961.895, "dur": 1.132,
    "args": {
      "External id": 2481885,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656651970.120, "dur": 128.137,
    "args": {
      "External id": 2481886,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656651971.742, "dur": 6.305,
    "args": {
      "External id": 2481887,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656651974.020, "dur": 3.156,
    "args": {
      "External id": 2481888,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656651975.596, "dur": 1.289,
    "args": {
      "External id": 2481889,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656651979.236, "dur": 118.124,
    "args": {
      "External id": 2481890,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652007.123, "dur": 89.230,
    "args": {
      "External id": 2481891,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652103.289, "dur": 5.386,
    "args": {
      "External id": 2481892,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652106.055, "dur": 0.990,
    "args": {
      "External id": 2481893,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652116.549, "dur": 2.221,
    "args": {
      "External id": 2481894,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652128.018, "dur": 6.115,
    "args": {
      "External id": 2481895,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652129.837, "dur": 4.044,
    "args": {
      "External id": 2481896,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652264.052, "dur": 193.118,
    "args": {
      "External id": 2481897,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652266.779, "dur": 2.400,
    "args": {
      "External id": 2481898,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652273.361, "dur": 183.369,
    "args": {
      "External id": 2481899,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652275.025, "dur": 0.483,
    "args": {
      "External id": 2481900,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652276.779, "dur": 25.374,
    "args": {
      "External id": 2481901,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652303.860, "dur": 5.258,
    "args": {
      "External id": 2481902,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652306.135, "dur": 2.698,
    "args": {
      "External id": 2481903,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652310.107, "dur": 24.210,
    "args": {
      "External id": 2481904,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652311.372, "dur": 1.794,
    "args": {
      "External id": 2481905,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652315.987, "dur": 18.063,
    "args": {
      "External id": 2481906,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652319.899, "dur": 2.911,
    "args": {
      "External id": 2481907,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652336.169, "dur": 21.688,
    "args": {
      "External id": 2481908,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652359.840, "dur": 13.490,
    "args": {
      "External id": 2481909,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652376.304, "dur": 15.395,
    "args": {
      "External id": 2481910,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652393.414, "dur": 12.857,
    "args": {
      "External id": 2481911,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652408.095, "dur": 21.656,
    "args": {
      "External id": 2481912,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652410.301, "dur": 1.386,
    "args": {
      "External id": 2481913,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652415.554, "dur": 1.235,
    "args": {
      "External id": 2481914,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652431.252, "dur": 12.303,
    "args": {
      "External id": 2481915,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652444.809, "dur": 10.804,
    "args": {
      "External id": 2481916,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652464.079, "dur": 2.041,
    "args": {
      "External id": 2481917,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652472.998, "dur": 3.931,
    "args": {
      "External id": 2481918,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652475.121, "dur": 0.876,
    "args": {
      "External id": 2481919,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652567.105, "dur": 67.738,
    "args": {
      "External id": 2481920,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652640.414, "dur": 10.354,
    "args": {
      "External id": 2481921,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652645.294, "dur": 2.613,
    "args": {
      "External id": 2481922,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652652.614, "dur": 26.414,
    "args": {
      "External id": 2481923,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652684.883, "dur": 5.719,
    "args": {
      "External id": 2481924,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652686.589, "dur": 3.258,
    "args": {
      "External id": 2481925,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652688.612, "dur": 0.965,
    "args": {
      "External id": 2481926,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652693.345, "dur": 44.242,
    "args": {
      "External id": 2481927,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652696.427, "dur": 40.602,
    "args": {
      "External id": 2481928,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652742.490, "dur": 15.532,
    "args": {
      "External id": 2481929,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652764.027, "dur": 3.683,
    "args": {
      "External id": 2481930,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652766.076, "dur": 0.712,
    "args": {
      "External id": 2481931,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652771.935, "dur": 48.776,
    "args": {
      "External id": 2481932,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652772.803, "dur": 4.835,
    "args": {
      "External id": 2481933,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652773.597, "dur": 3.545,
    "args": {
      "External id": 2481934,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652776.220, "dur": 0.796,
    "args": {
      "External id": 2481935,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652778.360, "dur": 41.859,
    "args": {
      "External id": 2481936,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652779.056, "dur": 40.649,
    "args": {
      "External id": 2481937,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652824.566, "dur": 3.898,
    "args": {
      "External id": 2481938,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652826.669, "dur": 0.744,
    "args": {
      "External id": 2481939,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652834.583, "dur": 1.739,
    "args": {
      "External id": 2481940,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652845.388, "dur": 6.578,
    "args": {
      "External id": 2481941,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652847.253, "dur": 4.435,
    "args": {
      "External id": 2481942,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652944.754, "dur": 225.085,
    "args": {
      "External id": 2481943,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652946.748, "dur": 3.725,
    "args": {
      "External id": 2481944,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652952.204, "dur": 217.186,
    "args": {
      "External id": 2481945,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652953.605, "dur": 0.310,
    "args": {
      "External id": 2481946,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652956.661, "dur": 20.251,
    "args": {
      "External id": 2481947,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656652978.559, "dur": 38.458,
    "args": {
      "External id": 2481948,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653014.815, "dur": 1.408,
    "args": {
      "External id": 2481949,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653018.593, "dur": 24.432,
    "args": {
      "External id": 2481950,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653019.892, "dur": 2.433,
    "args": {
      "External id": 2481951,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653023.560, "dur": 19.155,
    "args": {
      "External id": 2481952,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653026.484, "dur": 3.239,
    "args": {
      "External id": 2481953,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653044.536, "dur": 20.598,
    "args": {
      "External id": 2481954,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653066.753, "dur": 14.138,
    "args": {
      "External id": 2481955,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653083.865, "dur": 21.371,
    "args": {
      "External id": 2481956,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653108.893, "dur": 12.645,
    "args": {
      "External id": 2481957,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653123.030, "dur": 21.303,
    "args": {
      "External id": 2481958,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653126.679, "dur": 1.385,
    "args": {
      "External id": 2481959,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653130.359, "dur": 1.039,
    "args": {
      "External id": 2481960,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653145.822, "dur": 11.378,
    "args": {
      "External id": 2481961,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653158.256, "dur": 9.927,
    "args": {
      "External id": 2481962,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653178.849, "dur": 2.573,
    "args": {
      "External id": 2481963,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653192.955, "dur": 3.260,
    "args": {
      "External id": 2481964,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653195.123, "dur": 0.420,
    "args": {
      "External id": 2481965,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653267.096, "dur": 53.227,
    "args": {
      "External id": 2481966,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653324.888, "dur": 4.761,
    "args": {
      "External id": 2481967,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653327.569, "dur": 1.096,
    "args": {
      "External id": 2481968,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653331.119, "dur": 23.774,
    "args": {
      "External id": 2481969,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653359.235, "dur": 7.687,
    "args": {
      "External id": 2481970,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653360.801, "dur": 5.516,
    "args": {
      "External id": 2481971,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653364.414, "dur": 1.667,
    "args": {
      "External id": 2481972,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653369.596, "dur": 40.236,
    "args": {
      "External id": 2481973,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653370.732, "dur": 38.471,
    "args": {
      "External id": 2481974,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653414.142, "dur": 13.053,
    "args": {
      "External id": 2481975,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653433.129, "dur": 3.568,
    "args": {
      "External id": 2481976,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653435.032, "dur": 0.780,
    "args": {
      "External id": 2481977,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653441.993, "dur": 45.273,
    "args": {
      "External id": 2481978,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653442.897, "dur": 3.092,
    "args": {
      "External id": 2481979,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653443.483, "dur": 2.034,
    "args": {
      "External id": 2481980,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653444.670, "dur": 0.696,
    "args": {
      "External id": 2481981,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653446.509, "dur": 40.314,
    "args": {
      "External id": 2481982,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653447.356, "dur": 38.890,
    "args": {
      "External id": 2481983,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653506.328, "dur": 3.835,
    "args": {
      "External id": 2481984,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653508.412, "dur": 0.723,
    "args": {
      "External id": 2481985,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653517.396, "dur": 1.425,
    "args": {
      "External id": 2481986,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653526.161, "dur": 24.670,
    "args": {
      "External id": 2481987,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653543.563, "dur": 6.743,
    "args": {
      "External id": 2481988,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653640.263, "dur": 175.777,
    "args": {
      "External id": 2481989,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653642.521, "dur": 2.194,
    "args": {
      "External id": 2481990,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653646.387, "dur": 169.179,
    "args": {
      "External id": 2481991,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653650.067, "dur": 0.281,
    "args": {
      "External id": 2481992,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653651.497, "dur": 22.936,
    "args": {
      "External id": 2481993,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653676.023, "dur": 3.340,
    "args": {
      "External id": 2481994,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653677.927, "dur": 1.122,
    "args": {
      "External id": 2481995,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653680.166, "dur": 24.372,
    "args": {
      "External id": 2481996,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653681.328, "dur": 1.617,
    "args": {
      "External id": 2481997,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653683.965, "dur": 20.104,
    "args": {
      "External id": 2481998,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653688.358, "dur": 2.405,
    "args": {
      "External id": 2481999,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653706.124, "dur": 19.065,
    "args": {
      "External id": 2482000,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653726.430, "dur": 11.832,
    "args": {
      "External id": 2482001,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653741.034, "dur": 13.701,
    "args": {
      "External id": 2482002,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653758.279, "dur": 11.129,
    "args": {
      "External id": 2482003,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653771.197, "dur": 18.466,
    "args": {
      "External id": 2482004,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653772.892, "dur": 1.568,
    "args": {
      "External id": 2482005,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653776.802, "dur": 0.822,
    "args": {
      "External id": 2482006,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653791.043, "dur": 10.633,
    "args": {
      "External id": 2482007,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653802.673, "dur": 10.297,
    "args": {
      "External id": 2482008,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653823.042, "dur": 2.043,
    "args": {
      "External id": 2482009,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653833.507, "dur": 3.618,
    "args": {
      "External id": 2482010,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653835.873, "dur": 0.445,
    "args": {
      "External id": 2482011,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653896.198, "dur": 52.706,
    "args": {
      "External id": 2482012,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653953.565, "dur": 4.169,
    "args": {
      "External id": 2482013,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653955.972, "dur": 0.736,
    "args": {
      "External id": 2482014,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656653959.058, "dur": 70.298,
    "args": {
      "External id": 2482015,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654036.360, "dur": 7.971,
    "args": {
      "External id": 2482016,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654039.454, "dur": 3.929,
    "args": {
      "External id": 2482017,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654041.552, "dur": 1.585,
    "args": {
      "External id": 2482018,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654051.141, "dur": 84.086,
    "args": {
      "External id": 2482019,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654052.114, "dur": 82.395,
    "args": {
      "External id": 2482020,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654139.551, "dur": 22.558,
    "args": {
      "External id": 2482021,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654168.195, "dur": 5.773,
    "args": {
      "External id": 2482022,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654171.933, "dur": 1.078,
    "args": {
      "External id": 2482023,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654177.896, "dur": 47.710,
    "args": {
      "External id": 2482024,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654178.758, "dur": 3.295,
    "args": {
      "External id": 2482025,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654179.527, "dur": 2.023,
    "args": {
      "External id": 2482026,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654180.649, "dur": 0.756,
    "args": {
      "External id": 2482027,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654182.640, "dur": 42.596,
    "args": {
      "External id": 2482028,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654183.354, "dur": 41.336,
    "args": {
      "External id": 2482029,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654231.270, "dur": 5.485,
    "args": {
      "External id": 2482030,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654232.996, "dur": 2.506,
    "args": {
      "External id": 2482031,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654242.791, "dur": 1.558,
    "args": {
      "External id": 2482032,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654252.006, "dur": 6.378,
    "args": {
      "External id": 2482033,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654254.012, "dur": 4.107,
    "args": {
      "External id": 2482034,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654345.525, "dur": 166.144,
    "args": {
      "External id": 2482035,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654347.339, "dur": 2.134,
    "args": {
      "External id": 2482036,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654350.764, "dur": 160.327,
    "args": {
      "External id": 2482037,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654352.063, "dur": 0.415,
    "args": {
      "External id": 2482038,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654353.459, "dur": 22.306,
    "args": {
      "External id": 2482039,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654377.293, "dur": 3.369,
    "args": {
      "External id": 2482040,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654379.559, "dur": 0.817,
    "args": {
      "External id": 2482041,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654381.452, "dur": 22.903,
    "args": {
      "External id": 2482042,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654382.322, "dur": 1.382,
    "args": {
      "External id": 2482043,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654386.863, "dur": 17.154,
    "args": {
      "External id": 2482044,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654391.011, "dur": 2.336,
    "args": {
      "External id": 2482045,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654405.623, "dur": 19.026,
    "args": {
      "External id": 2482046,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654425.916, "dur": 12.095,
    "args": {
      "External id": 2482047,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654440.952, "dur": 12.523,
    "args": {
      "External id": 2482048,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654454.927, "dur": 11.035,
    "args": {
      "External id": 2482049,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654467.800, "dur": 17.236,
    "args": {
      "External id": 2482050,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654469.395, "dur": 1.243,
    "args": {
      "External id": 2482051,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654472.818, "dur": 0.844,
    "args": {
      "External id": 2482052,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654487.825, "dur": 11.073,
    "args": {
      "External id": 2482053,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654499.884, "dur": 10.288,
    "args": {
      "External id": 2482054,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654517.838, "dur": 1.766,
    "args": {
      "External id": 2482055,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654526.278, "dur": 18.944,
    "args": {
      "External id": 2482056,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654543.216, "dur": 0.670,
    "args": {
      "External id": 2482057,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654612.181, "dur": 51.079,
    "args": {
      "External id": 2482058,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654668.254, "dur": 6.195,
    "args": {
      "External id": 2482059,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654672.105, "dur": 1.407,
    "args": {
      "External id": 2482060,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654675.663, "dur": 22.966,
    "args": {
      "External id": 2482061,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654702.932, "dur": 4.442,
    "args": {
      "External id": 2482062,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654704.179, "dur": 2.627,
    "args": {
      "External id": 2482063,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654705.877, "dur": 0.737,
    "args": {
      "External id": 2482064,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654709.641, "dur": 37.353,
    "args": {
      "External id": 2482065,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654710.807, "dur": 35.576,
    "args": {
      "External id": 2482066,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654752.502, "dur": 13.921,
    "args": {
      "External id": 2482067,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654771.560, "dur": 3.312,
    "args": {
      "External id": 2482068,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654773.492, "dur": 0.552,
    "args": {
      "External id": 2482069,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654778.669, "dur": 48.140,
    "args": {
      "External id": 2482070,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654779.381, "dur": 6.997,
    "args": {
      "External id": 2482071,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654780.061, "dur": 5.747,
    "args": {
      "External id": 2482072,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654784.958, "dur": 0.727,
    "args": {
      "External id": 2482073,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654787.023, "dur": 39.361,
    "args": {
      "External id": 2482074,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654787.691, "dur": 38.089,
    "args": {
      "External id": 2482075,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654836.614, "dur": 3.908,
    "args": {
      "External id": 2482076,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654838.643, "dur": 0.781,
    "args": {
      "External id": 2482077,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654846.211, "dur": 1.721,
    "args": {
      "External id": 2482078,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654854.720, "dur": 7.745,
    "args": {
      "External id": 2482079,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654858.220, "dur": 3.987,
    "args": {
      "External id": 2482080,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654939.212, "dur": 206.465,
    "args": {
      "External id": 2482081,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654943.109, "dur": 1.902,
    "args": {
      "External id": 2482082,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654946.050, "dur": 198.948,
    "args": {
      "External id": 2482083,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654946.993, "dur": 0.417,
    "args": {
      "External id": 2482084,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654948.437, "dur": 19.353,
    "args": {
      "External id": 2482085,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654969.127, "dur": 5.216,
    "args": {
      "External id": 2482086,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654973.022, "dur": 0.926,
    "args": {
      "External id": 2482087,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654975.183, "dur": 58.513,
    "args": {
      "External id": 2482088,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656654977.926, "dur": 1.639,
    "args": {
      "External id": 2482089,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655013.838, "dur": 19.519,
    "args": {
      "External id": 2482090,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655017.650, "dur": 2.924,
    "args": {
      "External id": 2482091,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655034.990, "dur": 20.502,
    "args": {
      "External id": 2482092,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655056.800, "dur": 12.230,
    "args": {
      "External id": 2482093,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655071.817, "dur": 12.500,
    "args": {
      "External id": 2482094,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655085.626, "dur": 11.220,
    "args": {
      "External id": 2482095,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655098.325, "dur": 22.462,
    "args": {
      "External id": 2482096,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655102.440, "dur": 1.016,
    "args": {
      "External id": 2482097,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655107.058, "dur": 0.878,
    "args": {
      "External id": 2482098,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655121.982, "dur": 10.923,
    "args": {
      "External id": 2482099,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655133.798, "dur": 10.254,
    "args": {
      "External id": 2482100,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655153.893, "dur": 2.424,
    "args": {
      "External id": 2482101,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655164.659, "dur": 3.298,
    "args": {
      "External id": 2482102,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655166.670, "dur": 0.527,
    "args": {
      "External id": 2482103,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655234.644, "dur": 55.187,
    "args": {
      "External id": 2482104,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655294.751, "dur": 6.513,
    "args": {
      "External id": 2482105,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655299.064, "dur": 1.102,
    "args": {
      "External id": 2482106,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655302.726, "dur": 23.602,
    "args": {
      "External id": 2482107,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655330.847, "dur": 5.370,
    "args": {
      "External id": 2482108,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655332.276, "dur": 3.306,
    "args": {
      "External id": 2482109,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655333.818, "dur": 1.533,
    "args": {
      "External id": 2482110,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655338.864, "dur": 40.183,
    "args": {
      "External id": 2482111,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655341.786, "dur": 36.616,
    "args": {
      "External id": 2482112,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655383.421, "dur": 13.400,
    "args": {
      "External id": 2482113,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655402.343, "dur": 3.615,
    "args": {
      "External id": 2482114,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655404.530, "dur": 0.674,
    "args": {
      "External id": 2482115,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655409.554, "dur": 54.739,
    "args": {
      "External id": 2482116,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655417.574, "dur": 5.651,
    "args": {
      "External id": 2482117,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655420.544, "dur": 2.184,
    "args": {
      "External id": 2482118,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655421.991, "dur": 0.563,
    "args": {
      "External id": 2482119,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655423.762, "dur": 40.076,
    "args": {
      "External id": 2482120,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655424.546, "dur": 38.661,
    "args": {
      "External id": 2482121,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655468.308, "dur": 3.752,
    "args": {
      "External id": 2482122,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655470.440, "dur": 0.604,
    "args": {
      "External id": 2482123,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655479.210, "dur": 1.444,
    "args": {
      "External id": 2482124,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655487.974, "dur": 7.350,
    "args": {
      "External id": 2482125,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655489.808, "dur": 5.212,
    "args": {
      "External id": 2482126,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655597.358, "dur": 177.563,
    "args": {
      "External id": 2482127,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655599.532, "dur": 3.107,
    "args": {
      "External id": 2482128,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655604.231, "dur": 170.257,
    "args": {
      "External id": 2482129,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655607.844, "dur": 0.400,
    "args": {
      "External id": 2482130,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655609.222, "dur": 23.018,
    "args": {
      "External id": 2482131,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655633.878, "dur": 2.995,
    "args": {
      "External id": 2482132,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655635.773, "dur": 0.729,
    "args": {
      "External id": 2482133,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655637.689, "dur": 21.759,
    "args": {
      "External id": 2482134,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655638.903, "dur": 1.479,
    "args": {
      "External id": 2482135,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655641.603, "dur": 17.489,
    "args": {
      "External id": 2482136,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655645.637, "dur": 2.832,
    "args": {
      "External id": 2482137,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655660.986, "dur": 22.328,
    "args": {
      "External id": 2482138,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655684.653, "dur": 13.185,
    "args": {
      "External id": 2482139,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655700.783, "dur": 13.536,
    "args": {
      "External id": 2482140,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655717.466, "dur": 11.412,
    "args": {
      "External id": 2482141,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655730.292, "dur": 17.740,
    "args": {
      "External id": 2482142,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655732.100, "dur": 1.606,
    "args": {
      "External id": 2482143,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655735.704, "dur": 0.788,
    "args": {
      "External id": 2482144,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655749.284, "dur": 10.752,
    "args": {
      "External id": 2482145,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655760.991, "dur": 10.974,
    "args": {
      "External id": 2482146,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655781.623, "dur": 2.039,
    "args": {
      "External id": 2482147,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655792.116, "dur": 3.273,
    "args": {
      "External id": 2482148,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655794.181, "dur": 0.423,
    "args": {
      "External id": 2482149,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655856.473, "dur": 51.589,
    "args": {
      "External id": 2482150,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655912.431, "dur": 4.948,
    "args": {
      "External id": 2482151,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655915.055, "dur": 1.225,
    "args": {
      "External id": 2482152,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655918.664, "dur": 21.664,
    "args": {
      "External id": 2482153,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655944.614, "dur": 6.695,
    "args": {
      "External id": 2482154,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655947.888, "dur": 2.846,
    "args": {
      "External id": 2482155,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655949.534, "dur": 0.993,
    "args": {
      "External id": 2482156,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655953.872, "dur": 76.512,
    "args": {
      "External id": 2482157,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656655955.017, "dur": 74.064,
    "args": {
      "External id": 2482158,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656036.424, "dur": 15.491,
    "args": {
      "External id": 2482159,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656058.472, "dur": 6.432,
    "args": {
      "External id": 2482160,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656062.686, "dur": 1.182,
    "args": {
      "External id": 2482161,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656068.852, "dur": 50.727,
    "args": {
      "External id": 2482162,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656069.710, "dur": 5.321,
    "args": {
      "External id": 2482163,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656070.699, "dur": 3.782,
    "args": {
      "External id": 2482164,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656071.973, "dur": 2.314,
    "args": {
      "External id": 2482165,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656075.655, "dur": 43.419,
    "args": {
      "External id": 2482166,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656076.415, "dur": 42.094,
    "args": {
      "External id": 2482167,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656123.740, "dur": 3.819,
    "args": {
      "External id": 2482168,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656125.758, "dur": 0.804,
    "args": {
      "External id": 2482169,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656135.501, "dur": 1.577,
    "args": {
      "External id": 2482170,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656144.619, "dur": 5.555,
    "args": {
      "External id": 2482171,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656146.381, "dur": 3.555,
    "args": {
      "External id": 2482172,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656230.877, "dur": 197.241,
    "args": {
      "External id": 2482173,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656233.285, "dur": 2.277,
    "args": {
      "External id": 2482174,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656238.626, "dur": 189.036,
    "args": {
      "External id": 2482175,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656239.962, "dur": 0.272,
    "args": {
      "External id": 2482176,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656272.879, "dur": 21.844,
    "args": {
      "External id": 2482177,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656296.302, "dur": 3.229,
    "args": {
      "External id": 2482178,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656298.096, "dur": 1.204,
    "args": {
      "External id": 2482179,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656300.275, "dur": 20.216,
    "args": {
      "External id": 2482180,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656301.045, "dur": 2.951,
    "args": {
      "External id": 2482181,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656305.086, "dur": 15.093,
    "args": {
      "External id": 2482182,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656307.122, "dur": 2.559,
    "args": {
      "External id": 2482183,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656323.347, "dur": 19.388,
    "args": {
      "External id": 2482184,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656344.226, "dur": 11.948,
    "args": {
      "External id": 2482185,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656358.674, "dur": 12.553,
    "args": {
      "External id": 2482186,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656372.471, "dur": 11.759,
    "args": {
      "External id": 2482187,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656385.595, "dur": 17.189,
    "args": {
      "External id": 2482188,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656387.341, "dur": 1.028,
    "args": {
      "External id": 2482189,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656390.263, "dur": 0.736,
    "args": {
      "External id": 2482190,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656403.919, "dur": 10.446,
    "args": {
      "External id": 2482191,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656417.141, "dur": 9.552,
    "args": {
      "External id": 2482192,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656434.541, "dur": 1.638,
    "args": {
      "External id": 2482193,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656443.974, "dur": 3.071,
    "args": {
      "External id": 2482194,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656446.002, "dur": 0.288,
    "args": {
      "External id": 2482195,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656504.651, "dur": 66.667,
    "args": {
      "External id": 2482196,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656577.592, "dur": 5.459,
    "args": {
      "External id": 2482197,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656580.569, "dur": 1.118,
    "args": {
      "External id": 2482198,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656584.325, "dur": 24.475,
    "args": {
      "External id": 2482199,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656616.028, "dur": 6.103,
    "args": {
      "External id": 2482200,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656617.324, "dur": 4.072,
    "args": {
      "External id": 2482201,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656618.815, "dur": 2.380,
    "args": {
      "External id": 2482202,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656625.214, "dur": 40.812,
    "args": {
      "External id": 2482203,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656626.268, "dur": 39.178,
    "args": {
      "External id": 2482204,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656670.246, "dur": 14.106,
    "args": {
      "External id": 2482205,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656689.807, "dur": 5.385,
    "args": {
      "External id": 2482206,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656693.638, "dur": 0.653,
    "args": {
      "External id": 2482207,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656698.718, "dur": 46.707,
    "args": {
      "External id": 2482208,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 9911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656699.634, "dur": 3.290,
    "args": {
      "External id": 2482209,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656700.376, "dur": 1.981,
    "args": {
      "External id": 2482210,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 9913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656701.492, "dur": 0.714,
    "args": {
      "External id": 2482211,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 9914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656703.544, "dur": 41.478,
    "args": {
      "External id": 2482212,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656706.073, "dur": 38.413,
    "args": {
      "External id": 2482213,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 9916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656749.561, "dur": 3.822,
    "args": {
      "External id": 2482214,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656751.573, "dur": 0.776,
    "args": {
      "External id": 2482215,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656758.570, "dur": 1.610,
    "args": {
      "External id": 2482216,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 9919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656767.003, "dur": 7.978,
    "args": {
      "External id": 2482217,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 9920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656770.527, "dur": 4.185,
    "args": {
      "External id": 2482218,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656852.813, "dur": 204.831,
    "args": {
      "External id": 2482219,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656854.530, "dur": 1.800,
    "args": {
      "External id": 2482220,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656859.322, "dur": 197.703,
    "args": {
      "External id": 2482221,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 9924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656860.709, "dur": 0.240,
    "args": {
      "External id": 2482222,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656861.891, "dur": 18.988,
    "args": {
      "External id": 2482223,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 9926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656882.165, "dur": 4.636,
    "args": {
      "External id": 2482224,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 9927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656884.018, "dur": 2.510,
    "args": {
      "External id": 2482225,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 9928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656887.629, "dur": 24.125,
    "args": {
      "External id": 2482226,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656894.281, "dur": 1.264,
    "args": {
      "External id": 2482227,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656896.565, "dur": 14.875,
    "args": {
      "External id": 2482228,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 9931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656898.514, "dur": 2.360,
    "args": {
      "External id": 2482229,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656913.162, "dur": 18.694,
    "args": {
      "External id": 2482230,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656933.055, "dur": 12.294,
    "args": {
      "External id": 2482231,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656947.922, "dur": 12.068,
    "args": {
      "External id": 2482232,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 9935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656961.147, "dur": 11.037,
    "args": {
      "External id": 2482233,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 9936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656973.798, "dur": 56.401,
    "args": {
      "External id": 2482234,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 9937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656977.050, "dur": 1.117,
    "args": {
      "External id": 2482235,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 9938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656656982.279, "dur": 0.643,
    "args": {
      "External id": 2482236,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 9939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657032.838, "dur": 11.580,
    "args": {
      "External id": 2482237,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 9940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657045.346, "dur": 10.361,
    "args": {
      "External id": 2482238,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 9941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657065.620, "dur": 2.579,
    "args": {
      "External id": 2482239,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 9942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657076.805, "dur": 3.492,
    "args": {
      "External id": 2482240,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 9943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657078.935, "dur": 0.558,
    "args": {
      "External id": 2482241,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657143.205, "dur": 52.913,
    "args": {
      "External id": 2482242,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 9945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657200.740, "dur": 5.842,
    "args": {
      "External id": 2482243,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 9946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657204.799, "dur": 0.794,
    "args": {
      "External id": 2482244,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 9947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657207.839, "dur": 23.818,
    "args": {
      "External id": 2482245,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 9948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657235.825, "dur": 5.326,
    "args": {
      "External id": 2482246,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 9949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657237.104, "dur": 3.480,
    "args": {
      "External id": 2482247,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 9950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657238.964, "dur": 1.440,
    "args": {
      "External id": 2482248,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 9951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657243.669, "dur": 42.283,
    "args": {
      "External id": 2482249,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657246.331, "dur": 38.971,
    "args": {
      "External id": 2482250,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 9953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657290.004, "dur": 13.278,
    "args": {
      "External id": 2482251,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657307.783, "dur": 22.790,
    "args": {
      "External id": 2482252,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 9955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657310.065, "dur": 20.076,
    "args": {
      "External id": 2482253,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 9956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657315.127, "dur": 0.835,
    "args": {
      "External id": 2482254,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 9957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657336.743, "dur": 29.192,
    "args": {
      "External id": 2482255,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 9958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657338.365, "dur": 27.299,
    "args": {
      "External id": 2482256,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 9959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657342.843, "dur": 4.127,
    "args": {
      "External id": 2482257,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657349.678, "dur": 15.402,
    "args": {
      "External id": 2482258,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 9961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657377.941, "dur": 4.903,
    "args": {
      "External id": 2482259,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657379.574, "dur": 2.845,
    "args": {
      "External id": 2482260,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657383.942, "dur": 0.945,
    "args": {
      "External id": 2482261,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657384.344, "dur": 0.466,
    "args": {
      "External id": 2482262,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 9965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657425.607, "dur": 19.228,
    "args": {
      "External id": 2482263,"Sequence number": 24740640, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657446.811, "dur": 14.339,
    "args": {
      "External id": 2482264,"Sequence number": 24740641, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 9967
    }
  },
  {
    "ph": "s", "id": 9, "pid": 1336757, "tid": 1336757, "ts": 1514656657446.811,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657466.634, "dur": 5.815,
    "args": {
      "External id": 2482265,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], [], []], "Input Dims": [[16, 4, 4096], [], [], [], []], "Ev Idx": 9968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657469.998, "dur": 1.017,
    "args": {
      "External id": 2482266,"Record function id": 0, "Concrete Inputs": ["", "[16, 4, 4096]", "[8192, 1, 1]", "1"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657479.970, "dur": 7.412,
    "args": {
      "External id": 2482267,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "3"], "Input type": ["long int", "Scalar", "Scalar"], "Input Strides": [[8192, 1, 1], [], []], "Input Dims": [[16, 4, 4096], [], []], "Ev Idx": 9970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657485.634, "dur": 0.613,
    "args": {
      "External id": 2482268,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "4"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1, 1], [], [], []], "Input Dims": [[16, 4, 4096], [], [], []], "Ev Idx": 9971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657488.601, "dur": 2.782,
    "args": {
      "External id": 2482269,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], []], "Ev Idx": 9972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657489.887, "dur": 0.958,
    "args": {
      "External id": 2482270,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096]", "[8192, 1]", "4"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[8192, 1], [], [], []], "Input Dims": [[16, 4096], [], [], []], "Ev Idx": 9973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657497.258, "dur": 5.818,
    "args": {
      "External id": 2482271,"Sequence number": 24740642, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9974
    }
  },
  {
    "ph": "s", "id": 8, "pid": 1336757, "tid": 1336757, "ts": 1514656657497.258,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657500.414, "dur": 1.133,
    "args": {
      "External id": 2482272,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657504.064, "dur": 4.948,
    "args": {
      "External id": 2482273,"Sequence number": 24740643, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "1", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], [], []], "Ev Idx": 9976
    }
  },
  {
    "ph": "s", "id": 7, "pid": 1336757, "tid": 1336757, "ts": 1514656657504.064,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657507.307, "dur": 0.828,
    "args": {
      "External id": 2482274,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 4, 2048]", "[33554432, 8192, 2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::select", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657510.003, "dur": 6.174,
    "args": {
      "External id": 2482275,"Sequence number": 24740644, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "3"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], []], "Input Dims": [[16, 4096, 4, 2048], [], []], "Ev Idx": 9978
    }
  },
  {
    "ph": "s", "id": 6, "pid": 1336757, "tid": 1336757, "ts": 1514656657510.003,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657514.283, "dur": 0.854,
    "args": {
      "External id": 2482276,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 2048, 1], [], [], []], "Input Dims": [[16, 4096, 4, 2048], [], [], []], "Ev Idx": 9979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657517.181, "dur": 5.620,
    "args": {
      "External id": 2482277,"Sequence number": 24740645, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "2", "0", "9223372036854775807", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], [], []], "Ev Idx": 9980
    }
  },
  {
    "ph": "s", "id": 5, "pid": 1336757, "tid": 1336757, "ts": 1514656657517.181,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657519.541, "dur": 2.430,
    "args": {
      "External id": 2482278,"Record function id": 0, "Concrete Inputs": ["", "[16, 4096, 2048]", "[33554432, 8192, 1]", "6144"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[33554432, 8192, 1], [], [], []], "Input Dims": [[16, 4096, 2048], [], [], []], "Ev Idx": 9981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657526.379, "dur": 51.426,
    "args": {
      "External id": 2482279,"Sequence number": 24740646, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657545.256, "dur": 32.351,
    "args": {
      "External id": 2482280,"Sequence number": 24740646, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["long int", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657547.797, "dur": 7.658,
    "args": {
      "External id": 2482281,"Record function id": 0, "Concrete Inputs": ["", "4", "0", "", "", "0"], "Input type": ["long int", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[16, 4096], [], [], [], [], []], "Ev Idx": 9984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657550.358, "dur": 4.349,
    "args": {
      "External id": 2482282,"Record function id": 0, "Concrete Inputs": ["[16, 4096]", "4", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657556.307, "dur": 20.806,
    "args": {
      "External id": 2482283,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "long int", "Scalar"], "Input Strides": [[4096, 1], [8192, 1], []], "Input Dims": [[16, 4096], [16, 4096], []], "Ev Idx": 9986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657605.520, "dur": 5.473,
    "args": {
      "External id": 2482284,"Sequence number": 24740646, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1, 2048]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[33554432, 8192, 1], []], "Input Dims": [[16, 4096, 2048], []], "Ev Idx": 9987
    }
  },
  {
    "ph": "s", "id": 4, "pid": 1336757, "tid": 1336757, "ts": 1514656657605.520,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657613.340, "dur": 1.118,
    "args": {
      "External id": 2482285,"Sequence number": 24740647, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["long int", "ScalarList"], "Input Strides": [[4096, 1], []], "Input Dims": [[16, 4096], []], "Ev Idx": 9988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "FusedLinearCrossEntropyFunction", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657646.993, "dur": 43643.566,
    "args": {
      "External id": 2482286,"Sequence number": 24740647, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "", "", "-100", "0.", "1.", "8"], "Input type": ["c10::BFloat16", "long int", "c10::BFloat16", "", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[8192, 1], [1], [2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [65536], [32000, 2048], [], [], [], [], []], "Ev Idx": 9989
    }
  },
  {
    "ph": "s", "id": 3, "pid": 1336757, "tid": 1336757, "ts": 1514656657646.993,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::contiguous", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657661.074, "dur": 25.588,
    "args": {
      "External id": 2482287,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657661.743, "dur": 24.705,
    "args": {
      "External id": 2482288,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[8192, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657663.043, "dur": 6.680,
    "args": {
      "External id": 2482289,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "", "0"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[8192, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657664.282, "dur": 5.056,
    "args": {
      "External id": 2482290,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "15", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657670.421, "dur": 15.639,
    "args": {
      "External id": 2482291,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [8192, 1], []], "Input Dims": [[65536, 2048], [65536, 2048], []], "Ev Idx": 9994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657701.910, "dur": 25.400,
    "args": {
      "External id": 2482292,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", ""], "Input type": ["c10::BFloat16", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657702.916, "dur": 7.506,
    "args": {
      "External id": 2482293,"Record function id": 0, "Concrete Inputs": ["", "15", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], [], []], "Ev Idx": 9996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657706.487, "dur": 3.627,
    "args": {
      "External id": 2482294,"Record function id": 0, "Concrete Inputs": ["[65536, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 9997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657711.639, "dur": 15.453,
    "args": {
      "External id": 2482295,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 9998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657713.364, "dur": 13.162,
    "args": {
      "External id": 2482296,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[65536, 2048], []], "Ev Idx": 9999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657730.693, "dur": 17.461,
    "args": {
      "External id": 2482297,"Record function id": 0, "Concrete Inputs": ["", "6", "", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 10000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657731.470, "dur": 3.864,
    "args": {
      "External id": 2482298,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "False", ""], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 10001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657732.653, "dur": 2.434,
    "args": {
      "External id": 2482299,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657735.837, "dur": 12.122,
    "args": {
      "External id": 2482300,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657736.402, "dur": 11.117,
    "args": {
      "External id": 2482301,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[2048, 1], []], "Input Dims": [[32000, 2048], []], "Ev Idx": 10004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657753.976, "dur": 17.726,
    "args": {
      "External id": 2482302,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False"], "Input type": ["ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 10005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657757.280, "dur": 3.265,
    "args": {
      "External id": 2482303,"Record function id": 0, "Concrete Inputs": ["[65536]", "6", "", "", "False", ""], "Input type": ["ScalarList", "Scalar", "", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657761.225, "dur": 10.219,
    "args": {
      "External id": 2482304,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[65536]], "Ev Idx": 10007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657761.819, "dur": 9.340,
    "args": {
      "External id": 2482305,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657776.274, "dur": 20.067,
    "args": {
      "External id": 2482306,"Record function id": 0, "Concrete Inputs": ["", "-100"], "Input type": ["long int", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657799.165, "dur": 44.553,
    "args": {
      "External id": 2482307,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["bool", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657801.279, "dur": 41.948,
    "args": {
      "External id": 2482308,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["bool", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657805.715, "dur": 2.194,
    "args": {
      "External id": 2482309,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["long int", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657808.957, "dur": 20.508,
    "args": {
      "External id": 2482310,"Record function id": 0, "Concrete Inputs": ["", "4", "False", "False", ""], "Input type": ["bool", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657810.334, "dur": 18.731,
    "args": {
      "External id": 2482311,"Record function id": 0, "Concrete Inputs": ["", "4", "", "", "", "False", ""], "Input type": ["bool", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[1], [], [], [], [], [], []], "Input Dims": [[65536], [], [], [], [], [], []], "Ev Idx": 10014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657812.567, "dur": 2.676,
    "args": {
      "External id": 2482312,"Record function id": 0, "Concrete Inputs": ["[65536]", "[1]", "4", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657816.115, "dur": 12.574,
    "args": {
      "External id": 2482313,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["long int", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[65536], [65536], []], "Ev Idx": 10016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657849.139, "dur": 37893.047,
    "args": {
      "External id": 2482314,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 10017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336757, "tid": 1336757,
    "ts": 1514656657850.629, "dur": 37890.108,
    "args": {
      "External id": 2482315,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["long int"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 10018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656695753.729, "dur": 7.020,
    "args": {
      "External id": 2482316,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656695757.778, "dur": 1.110,
    "args": {
      "External id": 2482317,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656695765.709, "dur": 113.297,
    "args": {
      "External id": 2482318,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656695767.367, "dur": 9.044,
    "args": {
      "External id": 2482319,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656695769.514, "dur": 5.775,
    "args": {
      "External id": 2482320,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656695773.498, "dur": 1.411,
    "args": {
      "External id": 2482321,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656695777.984, "dur": 100.271,
    "args": {
      "External id": 2482322,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656695780.149, "dur": 96.964,
    "args": {
      "External id": 2482323,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656695883.069, "dur": 4.242,
    "args": {
      "External id": 2482324,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656695885.223, "dur": 0.644,
    "args": {
      "External id": 2482325,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656695895.342, "dur": 2.685,
    "args": {
      "External id": 2482326,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656695909.481, "dur": 6.605,
    "args": {
      "External id": 2482327,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656695911.284, "dur": 4.446,
    "args": {
      "External id": 2482328,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696079.470, "dur": 194.286,
    "args": {
      "External id": 2482329,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696084.414, "dur": 3.090,
    "args": {
      "External id": 2482330,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696089.311, "dur": 183.996,
    "args": {
      "External id": 2482331,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696091.129, "dur": 0.417,
    "args": {
      "External id": 2482332,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696092.943, "dur": 26.007,
    "args": {
      "External id": 2482333,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696122.684, "dur": 5.314,
    "args": {
      "External id": 2482334,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696126.610, "dur": 0.923,
    "args": {
      "External id": 2482335,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696129.189, "dur": 22.295,
    "args": {
      "External id": 2482336,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696130.231, "dur": 1.428,
    "args": {
      "External id": 2482337,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696132.875, "dur": 18.246,
    "args": {
      "External id": 2482338,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696136.578, "dur": 2.769,
    "args": {
      "External id": 2482339,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696153.244, "dur": 21.083,
    "args": {
      "External id": 2482340,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696176.187, "dur": 16.453,
    "args": {
      "External id": 2482341,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696196.125, "dur": 13.963,
    "args": {
      "External id": 2482342,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696211.549, "dur": 12.199,
    "args": {
      "External id": 2482343,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696227.185, "dur": 19.834,
    "args": {
      "External id": 2482344,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696229.463, "dur": 1.655,
    "args": {
      "External id": 2482345,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696233.315, "dur": 0.955,
    "args": {
      "External id": 2482346,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696248.690, "dur": 11.877,
    "args": {
      "External id": 2482347,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696261.933, "dur": 10.043,
    "args": {
      "External id": 2482348,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696281.282, "dur": 2.417,
    "args": {
      "External id": 2482349,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696291.988, "dur": 4.130,
    "args": {
      "External id": 2482350,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696294.486, "dur": 0.707,
    "args": {
      "External id": 2482351,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "0"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696373.710, "dur": 69.115,
    "args": {
      "External id": 2482352,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696447.960, "dur": 5.641,
    "args": {
      "External id": 2482353,"Record function id": 0, "Concrete Inputs": ["", "0", "0", "8192", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696450.552, "dur": 0.801,
    "args": {
      "External id": 2482354,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "0"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696455.114, "dur": 25.428,
    "args": {
      "External id": 2482355,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696485.651, "dur": 7.811,
    "args": {
      "External id": 2482356,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696487.559, "dur": 5.230,
    "args": {
      "External id": 2482357,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696491.361, "dur": 1.209,
    "args": {
      "External id": 2482358,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696496.833, "dur": 61.107,
    "args": {
      "External id": 2482359,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696498.033, "dur": 58.807,
    "args": {
      "External id": 2482360,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696564.127, "dur": 16.653,
    "args": {
      "External id": 2482361,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696587.976, "dur": 5.016,
    "args": {
      "External id": 2482362,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696590.614, "dur": 1.075,
    "args": {
      "External id": 2482363,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696598.974, "dur": 50.833,
    "args": {
      "External id": 2482364,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696600.035, "dur": 3.532,
    "args": {
      "External id": 2482365,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696600.933, "dur": 2.068,
    "args": {
      "External id": 2482366,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696602.159, "dur": 0.726,
    "args": {
      "External id": 2482367,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696604.180, "dur": 45.237,
    "args": {
      "External id": 2482368,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696605.146, "dur": 43.724,
    "args": {
      "External id": 2482369,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696655.448, "dur": 3.843,
    "args": {
      "External id": 2482370,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696657.312, "dur": 0.860,
    "args": {
      "External id": 2482371,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696665.447, "dur": 1.805,
    "args": {
      "External id": 2482372,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696675.234, "dur": 6.890,
    "args": {
      "External id": 2482373,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696676.935, "dur": 4.918,
    "args": {
      "External id": 2482374,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696779.264, "dur": 174.369,
    "args": {
      "External id": 2482375,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696781.465, "dur": 2.095,
    "args": {
      "External id": 2482376,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696785.202, "dur": 167.808,
    "args": {
      "External id": 2482377,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696786.916, "dur": 0.280,
    "args": {
      "External id": 2482378,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696788.020, "dur": 22.736,
    "args": {
      "External id": 2482379,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696812.416, "dur": 3.079,
    "args": {
      "External id": 2482380,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696814.671, "dur": 0.547,
    "args": {
      "External id": 2482381,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696816.529, "dur": 23.801,
    "args": {
      "External id": 2482382,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696817.769, "dur": 1.586,
    "args": {
      "External id": 2482383,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696820.582, "dur": 19.459,
    "args": {
      "External id": 2482384,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696826.470, "dur": 2.903,
    "args": {
      "External id": 2482385,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696842.003, "dur": 20.457,
    "args": {
      "External id": 2482386,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696864.126, "dur": 11.873,
    "args": {
      "External id": 2482387,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696879.036, "dur": 13.448,
    "args": {
      "External id": 2482388,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696893.779, "dur": 11.672,
    "args": {
      "External id": 2482389,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696907.030, "dur": 19.638,
    "args": {
      "External id": 2482390,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696909.144, "dur": 1.332,
    "args": {
      "External id": 2482391,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696912.742, "dur": 0.813,
    "args": {
      "External id": 2482392,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696929.943, "dur": 10.660,
    "args": {
      "External id": 2482393,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696941.821, "dur": 10.191,
    "args": {
      "External id": 2482394,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696961.027, "dur": 1.939,
    "args": {
      "External id": 2482395,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696970.676, "dur": 3.069,
    "args": {
      "External id": 2482396,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656696972.646, "dur": 0.408,
    "args": {
      "External id": 2482397,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "8192"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697088.518, "dur": 57.708,
    "args": {
      "External id": 2482398,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697151.530, "dur": 6.770,
    "args": {
      "External id": 2482399,"Record function id": 0, "Concrete Inputs": ["", "0", "8192", "16384", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697155.056, "dur": 1.827,
    "args": {
      "External id": 2482400,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "16777216"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697161.095, "dur": 24.021,
    "args": {
      "External id": 2482401,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697190.189, "dur": 5.622,
    "args": {
      "External id": 2482402,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697191.741, "dur": 3.437,
    "args": {
      "External id": 2482403,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697194.111, "dur": 0.865,
    "args": {
      "External id": 2482404,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697198.457, "dur": 39.950,
    "args": {
      "External id": 2482405,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697199.353, "dur": 38.296,
    "args": {
      "External id": 2482406,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697242.648, "dur": 14.475,
    "args": {
      "External id": 2482407,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697264.507, "dur": 3.736,
    "args": {
      "External id": 2482408,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697266.560, "dur": 0.928,
    "args": {
      "External id": 2482409,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697272.270, "dur": 49.776,
    "args": {
      "External id": 2482410,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697273.196, "dur": 5.209,
    "args": {
      "External id": 2482411,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697274.018, "dur": 3.879,
    "args": {
      "External id": 2482412,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697275.256, "dur": 2.415,
    "args": {
      "External id": 2482413,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697280.646, "dur": 41.021,
    "args": {
      "External id": 2482414,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697281.505, "dur": 39.390,
    "args": {
      "External id": 2482415,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697325.830, "dur": 3.658,
    "args": {
      "External id": 2482416,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697327.772, "dur": 0.640,
    "args": {
      "External id": 2482417,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697335.450, "dur": 1.620,
    "args": {
      "External id": 2482418,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697344.702, "dur": 6.220,
    "args": {
      "External id": 2482419,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697346.623, "dur": 4.033,
    "args": {
      "External id": 2482420,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697434.848, "dur": 192.272,
    "args": {
      "External id": 2482421,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697437.285, "dur": 1.987,
    "args": {
      "External id": 2482422,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697441.038, "dur": 185.645,
    "args": {
      "External id": 2482423,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697442.199, "dur": 0.381,
    "args": {
      "External id": 2482424,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697445.298, "dur": 23.525,
    "args": {
      "External id": 2482425,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697470.560, "dur": 2.929,
    "args": {
      "External id": 2482426,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697472.317, "dur": 0.853,
    "args": {
      "External id": 2482427,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697474.363, "dur": 23.606,
    "args": {
      "External id": 2482428,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697475.744, "dur": 3.167,
    "args": {
      "External id": 2482429,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697480.203, "dur": 17.363,
    "args": {
      "External id": 2482430,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697484.298, "dur": 2.608,
    "args": {
      "External id": 2482431,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697499.479, "dur": 18.181,
    "args": {
      "External id": 2482432,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697519.091, "dur": 28.640,
    "args": {
      "External id": 2482433,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697551.706, "dur": 14.167,
    "args": {
      "External id": 2482434,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697567.182, "dur": 11.477,
    "args": {
      "External id": 2482435,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697580.539, "dur": 19.280,
    "args": {
      "External id": 2482436,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697582.592, "dur": 1.724,
    "args": {
      "External id": 2482437,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697586.455, "dur": 0.869,
    "args": {
      "External id": 2482438,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697603.146, "dur": 11.281,
    "args": {
      "External id": 2482439,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697615.551, "dur": 10.129,
    "args": {
      "External id": 2482440,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697635.170, "dur": 2.478,
    "args": {
      "External id": 2482441,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697646.322, "dur": 3.706,
    "args": {
      "External id": 2482442,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697648.543, "dur": 0.417,
    "args": {
      "External id": 2482443,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "16384"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697717.451, "dur": 86.297,
    "args": {
      "External id": 2482444,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697808.611, "dur": 6.222,
    "args": {
      "External id": 2482445,"Record function id": 0, "Concrete Inputs": ["", "0", "16384", "24576", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697812.853, "dur": 0.944,
    "args": {
      "External id": 2482446,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "33554432"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697816.142, "dur": 28.177,
    "args": {
      "External id": 2482447,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697848.925, "dur": 6.380,
    "args": {
      "External id": 2482448,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697850.498, "dur": 4.165,
    "args": {
      "External id": 2482449,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697852.158, "dur": 2.284,
    "args": {
      "External id": 2482450,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697857.687, "dur": 73.853,
    "args": {
      "External id": 2482451,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697859.221, "dur": 71.654,
    "args": {
      "External id": 2482452,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697937.554, "dur": 24.105,
    "args": {
      "External id": 2482453,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697967.326, "dur": 3.789,
    "args": {
      "External id": 2482454,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697969.447, "dur": 0.590,
    "args": {
      "External id": 2482455,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697974.680, "dur": 91.290,
    "args": {
      "External id": 2482456,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697975.513, "dur": 5.050,
    "args": {
      "External id": 2482457,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697976.106, "dur": 3.991,
    "args": {
      "External id": 2482458,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656697979.227, "dur": 0.742,
    "args": {
      "External id": 2482459,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698015.986, "dur": 49.309,
    "args": {
      "External id": 2482460,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698016.903, "dur": 47.660,
    "args": {
      "External id": 2482461,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698070.975, "dur": 17.637,
    "args": {
      "External id": 2482462,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698086.219, "dur": 0.988,
    "args": {
      "External id": 2482463,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698095.358, "dur": 1.717,
    "args": {
      "External id": 2482464,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698106.651, "dur": 8.270,
    "args": {
      "External id": 2482465,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698108.615, "dur": 5.943,
    "args": {
      "External id": 2482466,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698206.898, "dur": 174.679,
    "args": {
      "External id": 2482467,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698208.989, "dur": 2.432,
    "args": {
      "External id": 2482468,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698213.160, "dur": 167.887,
    "args": {
      "External id": 2482469,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698214.767, "dur": 0.360,
    "args": {
      "External id": 2482470,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698216.222, "dur": 23.899,
    "args": {
      "External id": 2482471,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698243.514, "dur": 3.011,
    "args": {
      "External id": 2482472,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698245.571, "dur": 0.571,
    "args": {
      "External id": 2482473,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698247.460, "dur": 21.526,
    "args": {
      "External id": 2482474,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698248.431, "dur": 1.556,
    "args": {
      "External id": 2482475,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698251.279, "dur": 17.317,
    "args": {
      "External id": 2482476,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698255.751, "dur": 2.351,
    "args": {
      "External id": 2482477,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698270.718, "dur": 20.422,
    "args": {
      "External id": 2482478,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698292.663, "dur": 12.013,
    "args": {
      "External id": 2482479,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698307.749, "dur": 14.063,
    "args": {
      "External id": 2482480,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698323.040, "dur": 11.646,
    "args": {
      "External id": 2482481,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698337.880, "dur": 17.893,
    "args": {
      "External id": 2482482,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698339.583, "dur": 1.496,
    "args": {
      "External id": 2482483,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698342.947, "dur": 0.707,
    "args": {
      "External id": 2482484,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698356.875, "dur": 11.100,
    "args": {
      "External id": 2482485,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698369.128, "dur": 11.102,
    "args": {
      "External id": 2482486,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698388.053, "dur": 1.804,
    "args": {
      "External id": 2482487,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698397.458, "dur": 3.132,
    "args": {
      "External id": 2482488,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698399.520, "dur": 0.334,
    "args": {
      "External id": 2482489,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "24576"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698467.136, "dur": 48.456,
    "args": {
      "External id": 2482490,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698520.512, "dur": 4.565,
    "args": {
      "External id": 2482491,"Record function id": 0, "Concrete Inputs": ["", "0", "24576", "32768", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698522.885, "dur": 1.163,
    "args": {
      "External id": 2482492,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "50331648"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698526.497, "dur": 43.193,
    "args": {
      "External id": 2482493,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698576.152, "dur": 7.130,
    "args": {
      "External id": 2482494,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698577.602, "dur": 4.986,
    "args": {
      "External id": 2482495,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698581.441, "dur": 0.941,
    "args": {
      "External id": 2482496,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698586.262, "dur": 43.364,
    "args": {
      "External id": 2482497,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698587.367, "dur": 41.531,
    "args": {
      "External id": 2482498,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698633.853, "dur": 13.994,
    "args": {
      "External id": 2482499,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698653.629, "dur": 4.265,
    "args": {
      "External id": 2482500,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698655.864, "dur": 0.902,
    "args": {
      "External id": 2482501,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698661.784, "dur": 48.300,
    "args": {
      "External id": 2482502,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698664.397, "dur": 4.721,
    "args": {
      "External id": 2482503,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698665.065, "dur": 3.531,
    "args": {
      "External id": 2482504,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698666.349, "dur": 2.096,
    "args": {
      "External id": 2482505,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698669.767, "dur": 39.934,
    "args": {
      "External id": 2482506,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698670.675, "dur": 38.418,
    "args": {
      "External id": 2482507,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698714.208, "dur": 3.725,
    "args": {
      "External id": 2482508,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698716.242, "dur": 0.535,
    "args": {
      "External id": 2482509,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698723.873, "dur": 1.624,
    "args": {
      "External id": 2482510,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698734.856, "dur": 5.791,
    "args": {
      "External id": 2482511,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698736.668, "dur": 3.739,
    "args": {
      "External id": 2482512,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698823.614, "dur": 209.738,
    "args": {
      "External id": 2482513,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698825.878, "dur": 1.923,
    "args": {
      "External id": 2482514,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698829.243, "dur": 203.432,
    "args": {
      "External id": 2482515,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698830.303, "dur": 0.442,
    "args": {
      "External id": 2482516,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698835.134, "dur": 20.418,
    "args": {
      "External id": 2482517,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698861.378, "dur": 4.288,
    "args": {
      "External id": 2482518,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698864.576, "dur": 0.832,
    "args": {
      "External id": 2482519,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698866.465, "dur": 20.784,
    "args": {
      "External id": 2482520,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698867.608, "dur": 3.078,
    "args": {
      "External id": 2482521,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698871.821, "dur": 15.116,
    "args": {
      "External id": 2482522,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698874.036, "dur": 2.426,
    "args": {
      "External id": 2482523,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698888.614, "dur": 19.003,
    "args": {
      "External id": 2482524,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698908.752, "dur": 11.826,
    "args": {
      "External id": 2482525,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698924.727, "dur": 11.952,
    "args": {
      "External id": 2482526,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698938.085, "dur": 10.941,
    "args": {
      "External id": 2482527,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698950.790, "dur": 17.527,
    "args": {
      "External id": 2482528,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698952.387, "dur": 1.457,
    "args": {
      "External id": 2482529,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698955.953, "dur": 0.800,
    "args": {
      "External id": 2482530,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656698969.543, "dur": 47.090,
    "args": {
      "External id": 2482531,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699018.894, "dur": 12.386,
    "args": {
      "External id": 2482532,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699043.146, "dur": 2.347,
    "args": {
      "External id": 2482533,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699054.455, "dur": 4.099,
    "args": {
      "External id": 2482534,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699057.108, "dur": 0.635,
    "args": {
      "External id": 2482535,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "32768"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699124.365, "dur": 53.575,
    "args": {
      "External id": 2482536,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699182.427, "dur": 4.332,
    "args": {
      "External id": 2482537,"Record function id": 0, "Concrete Inputs": ["", "0", "32768", "40960", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699185.003, "dur": 0.838,
    "args": {
      "External id": 2482538,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "67108864"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699188.127, "dur": 21.378,
    "args": {
      "External id": 2482539,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699213.829, "dur": 6.780,
    "args": {
      "External id": 2482540,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699217.018, "dur": 2.984,
    "args": {
      "External id": 2482541,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699218.931, "dur": 0.823,
    "args": {
      "External id": 2482542,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699223.096, "dur": 38.665,
    "args": {
      "External id": 2482543,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699224.233, "dur": 36.894,
    "args": {
      "External id": 2482544,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699265.735, "dur": 13.128,
    "args": {
      "External id": 2482545,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699284.028, "dur": 5.215,
    "args": {
      "External id": 2482546,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699287.530, "dur": 0.708,
    "args": {
      "External id": 2482547,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699292.950, "dur": 45.696,
    "args": {
      "External id": 2482548,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699293.786, "dur": 5.410,
    "args": {
      "External id": 2482549,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699294.445, "dur": 4.164,
    "args": {
      "External id": 2482550,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699297.999, "dur": 0.458,
    "args": {
      "External id": 2482551,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699299.842, "dur": 38.301,
    "args": {
      "External id": 2482552,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699300.340, "dur": 37.153,
    "args": {
      "External id": 2482553,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699344.175, "dur": 4.080,
    "args": {
      "External id": 2482554,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699346.284, "dur": 0.891,
    "args": {
      "External id": 2482555,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699353.422, "dur": 1.447,
    "args": {
      "External id": 2482556,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699361.897, "dur": 6.230,
    "args": {
      "External id": 2482557,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699363.694, "dur": 4.184,
    "args": {
      "External id": 2482558,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699444.945, "dur": 189.491,
    "args": {
      "External id": 2482559,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699447.117, "dur": 1.880,
    "args": {
      "External id": 2482560,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699450.180, "dur": 183.825,
    "args": {
      "External id": 2482561,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699453.114, "dur": 0.292,
    "args": {
      "External id": 2482562,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699454.592, "dur": 24.780,
    "args": {
      "External id": 2482563,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699480.861, "dur": 3.103,
    "args": {
      "External id": 2482564,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699482.979, "dur": 0.595,
    "args": {
      "External id": 2482565,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699484.792, "dur": 22.438,
    "args": {
      "External id": 2482566,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699487.331, "dur": 1.372,
    "args": {
      "External id": 2482567,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699489.810, "dur": 17.122,
    "args": {
      "External id": 2482568,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699493.812, "dur": 2.639,
    "args": {
      "External id": 2482569,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699508.615, "dur": 33.686,
    "args": {
      "External id": 2482570,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699544.525, "dur": 14.349,
    "args": {
      "External id": 2482571,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699562.040, "dur": 13.099,
    "args": {
      "External id": 2482572,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699576.539, "dur": 10.854,
    "args": {
      "External id": 2482573,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699589.298, "dur": 19.017,
    "args": {
      "External id": 2482574,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699591.010, "dur": 1.755,
    "args": {
      "External id": 2482575,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699594.628, "dur": 0.732,
    "args": {
      "External id": 2482576,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699611.260, "dur": 10.269,
    "args": {
      "External id": 2482577,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699622.399, "dur": 10.807,
    "args": {
      "External id": 2482578,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699641.766, "dur": 2.271,
    "args": {
      "External id": 2482579,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699652.524, "dur": 3.612,
    "args": {
      "External id": 2482580,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699654.911, "dur": 0.425,
    "args": {
      "External id": 2482581,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "40960"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699719.240, "dur": 49.037,
    "args": {
      "External id": 2482582,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699772.586, "dur": 9.253,
    "args": {
      "External id": 2482583,"Record function id": 0, "Concrete Inputs": ["", "0", "40960", "49152", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699777.042, "dur": 3.620,
    "args": {
      "External id": 2482584,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "83886080"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699783.139, "dur": 21.793,
    "args": {
      "External id": 2482585,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699809.215, "dur": 4.946,
    "args": {
      "External id": 2482586,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699810.551, "dur": 2.883,
    "args": {
      "External id": 2482587,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699812.135, "dur": 1.152,
    "args": {
      "External id": 2482588,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699816.488, "dur": 37.865,
    "args": {
      "External id": 2482589,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699817.359, "dur": 36.214,
    "args": {
      "External id": 2482590,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699859.953, "dur": 13.148,
    "args": {
      "External id": 2482591,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699878.293, "dur": 3.588,
    "args": {
      "External id": 2482592,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699880.190, "dur": 0.777,
    "args": {
      "External id": 2482593,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699885.237, "dur": 46.579,
    "args": {
      "External id": 2482594,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699886.137, "dur": 5.146,
    "args": {
      "External id": 2482595,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699886.807, "dur": 4.029,
    "args": {
      "External id": 2482596,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699890.120, "dur": 0.598,
    "args": {
      "External id": 2482597,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699891.859, "dur": 39.515,
    "args": {
      "External id": 2482598,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699892.642, "dur": 38.278,
    "args": {
      "External id": 2482599,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699935.596, "dur": 3.833,
    "args": {
      "External id": 2482600,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699937.354, "dur": 1.044,
    "args": {
      "External id": 2482601,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699944.538, "dur": 1.411,
    "args": {
      "External id": 2482602,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699952.662, "dur": 8.153,
    "args": {
      "External id": 2482603,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10306
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656699956.412, "dur": 4.116,
    "args": {
      "External id": 2482604,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700078.875, "dur": 175.281,
    "args": {
      "External id": 2482605,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700081.366, "dur": 4.784,
    "args": {
      "External id": 2482606,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700087.601, "dur": 166.099,
    "args": {
      "External id": 2482607,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10310
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700088.994, "dur": 0.370,
    "args": {
      "External id": 2482608,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700090.675, "dur": 21.919,
    "args": {
      "External id": 2482609,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700113.908, "dur": 5.684,
    "args": {
      "External id": 2482610,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700118.070, "dur": 1.086,
    "args": {
      "External id": 2482611,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700120.732, "dur": 21.192,
    "args": {
      "External id": 2482612,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700123.543, "dur": 1.490,
    "args": {
      "External id": 2482613,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700126.182, "dur": 15.386,
    "args": {
      "External id": 2482614,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700128.824, "dur": 2.524,
    "args": {
      "External id": 2482615,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700143.298, "dur": 20.180,
    "args": {
      "External id": 2482616,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700165.070, "dur": 11.967,
    "args": {
      "External id": 2482617,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700179.611, "dur": 13.021,
    "args": {
      "External id": 2482618,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700193.972, "dur": 11.133,
    "args": {
      "External id": 2482619,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700206.670, "dur": 21.892,
    "args": {
      "External id": 2482620,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700209.921, "dur": 1.182,
    "args": {
      "External id": 2482621,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700215.308, "dur": 0.817,
    "args": {
      "External id": 2482622,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700230.183, "dur": 11.031,
    "args": {
      "External id": 2482623,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700242.221, "dur": 10.572,
    "args": {
      "External id": 2482624,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700261.122, "dur": 2.289,
    "args": {
      "External id": 2482625,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700272.199, "dur": 3.864,
    "args": {
      "External id": 2482626,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700274.537, "dur": 0.469,
    "args": {
      "External id": 2482627,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "49152"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700335.656, "dur": 55.011,
    "args": {
      "External id": 2482628,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700395.155, "dur": 6.098,
    "args": {
      "External id": 2482629,"Record function id": 0, "Concrete Inputs": ["", "0", "49152", "57344", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700399.273, "dur": 0.815,
    "args": {
      "External id": 2482630,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "100663296"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700402.540, "dur": 22.922,
    "args": {
      "External id": 2482631,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700429.932, "dur": 5.551,
    "args": {
      "External id": 2482632,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700431.326, "dur": 3.561,
    "args": {
      "External id": 2482633,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700433.064, "dur": 1.612,
    "args": {
      "External id": 2482634,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700438.036, "dur": 41.719,
    "args": {
      "External id": 2482635,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10338
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700441.331, "dur": 37.767,
    "args": {
      "External id": 2482636,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10339
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700484.239, "dur": 13.131,
    "args": {
      "External id": 2482637,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700502.705, "dur": 3.711,
    "args": {
      "External id": 2482638,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700504.689, "dur": 0.947,
    "args": {
      "External id": 2482639,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linear", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700510.067, "dur": 67.906,
    "args": {
      "External id": 2482640,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["c10::BFloat16", "c10::BFloat16", ""], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [32000, 2048], []], "Ev Idx": 10343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700510.794, "dur": 5.229,
    "args": {
      "External id": 2482641,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700511.568, "dur": 3.955,
    "args": {
      "External id": 2482642,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], []], "Input Dims": [[32000, 2048], [], []], "Ev Idx": 10345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700514.508, "dur": 0.887,
    "args": {
      "External id": 2482643,"Record function id": 0, "Concrete Inputs": ["", "[2048, 32000]", "[1, 2048]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[32000, 2048], [], [], []], "Ev Idx": 10346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700516.605, "dur": 60.665,
    "args": {
      "External id": 2482644,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700517.414, "dur": 58.767,
    "args": {
      "External id": 2482645,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[2048, 1], [1, 2048]], "Input Dims": [[8192, 2048], [2048, 32000]], "Ev Idx": 10348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700583.841, "dur": 4.717,
    "args": {
      "External id": 2482646,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["long int", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700586.341, "dur": 1.006,
    "args": {
      "External id": 2482647,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["long int", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700594.711, "dur": 1.660,
    "args": {
      "External id": 2482648,"Record function id": 0, "Concrete Inputs": ["", "[-1, 32000]"], "Input type": ["c10::BFloat16", "ScalarList"], "Input Strides": [[32000, 1], []], "Input Dims": [[8192, 32000], []], "Ev Idx": 10351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::new_empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700605.848, "dur": 6.848,
    "args": {
      "External id": 2482649,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "6", "", "", "False"], "Input type": ["c10::BFloat16", "ScalarList", "Scalar", "", "", "Scalar"], "Input Strides": [[32000, 1], [], [], [], [], []], "Input Dims": [[8192, 32000], [], [], [], [], []], "Ev Idx": 10352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700607.741, "dur": 4.688,
    "args": {
      "External id": 2482650,"Record function id": 0, "Concrete Inputs": ["[8192, 1]", "6", "0", "", "False", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "Scalar", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700694.011, "dur": 167.005,
    "args": {
      "External id": 2482651,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700697.926, "dur": 2.048,
    "args": {
      "External id": 2482652,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::logsumexp", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700701.126, "dur": 159.304,
    "args": {
      "External id": 2482653,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", "float"], "Input Strides": [[1, 1], [], [], [1]], "Input Dims": [[8192, 1], [], [], [0]], "Ev Idx": 10356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::real", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700702.345, "dur": 0.371,
    "args": {
      "External id": 2482654,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::amax", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700705.372, "dur": 20.245,
    "args": {
      "External id": 2482655,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "True"], "Input type": ["float", "ScalarList", "Scalar"], "Input Strides": [[1, 1], [], []], "Input Dims": [[8192, 1], [], []], "Ev Idx": 10358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::squeeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700727.057, "dur": 5.140,
    "args": {
      "External id": 2482656,"Record function id": 0, "Concrete Inputs": ["", "[-1]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1, 1], []], "Input Dims": [[8192, 1], []], "Ev Idx": 10359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700730.840, "dur": 1.043,
    "args": {
      "External id": 2482657,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1, 1], [], [], []], "Input Dims": [[8192, 1], [], [], []], "Ev Idx": 10360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700733.067, "dur": 19.582,
    "args": {
      "External id": 2482658,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700734.149, "dur": 1.446,
    "args": {
      "External id": 2482659,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700736.616, "dur": 15.750,
    "args": {
      "External id": 2482660,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[1], [1]], "Input Dims": [[8192], [0]], "Ev Idx": 10363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700739.491, "dur": 2.315,
    "args": {
      "External id": 2482661,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700753.984, "dur": 18.080,
    "args": {
      "External id": 2482662,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::masked_fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700773.426, "dur": 14.376,
    "args": {
      "External id": 2482663,"Record function id": 0, "Concrete Inputs": ["", "", "0"], "Input type": ["float", "bool", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sub", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700790.784, "dur": 12.565,
    "args": {
      "External id": 2482664,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1, 1], [1, 1], []], "Input Dims": [[8192, 1], [8192, 1], []], "Ev Idx": 10367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::exp_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700806.195, "dur": 10.844,
    "args": {
      "External id": 2482665,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1, 1]], "Input Dims": [[8192, 1]], "Ev Idx": 10368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700818.390, "dur": 17.755,
    "args": {
      "External id": 2482666,"Record function id": 0, "Concrete Inputs": ["", "[-1]", "False", "", ""], "Input type": ["float", "ScalarList", "Scalar", "", "float"], "Input Strides": [[1, 1], [], [], [], [1]], "Input Dims": [[8192, 1], [], [], [], [0]], "Ev Idx": 10369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700820.113, "dur": 1.239,
    "args": {
      "External id": 2482667,"Record function id": 0, "Concrete Inputs": ["", "[8192]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 10370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700823.110, "dur": 1.004,
    "args": {
      "External id": 2482668,"Record function id": 0, "Concrete Inputs": ["", "[8192, 1]", "[1, 0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[8192], [], [], []], "Ev Idx": 10371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::log_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700837.292, "dur": 10.875,
    "args": {
      "External id": 2482669,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[8192]], "Ev Idx": 10372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700849.071, "dur": 10.614,
    "args": {
      "External id": 2482670,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [1], []], "Input Dims": [[8192], [8192], []], "Ev Idx": 10373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700867.290, "dur": 1.622,
    "args": {
      "External id": 2482671,"Record function id": 0, "Concrete Inputs": ["", "[8192]"], "Input type": ["float", "ScalarList"], "Input Strides": [[1], []], "Input Dims": [[8192], []], "Ev Idx": 10374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700877.901, "dur": 3.124,
    "args": {
      "External id": 2482672,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["float", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], []], "Input Dims": [[65536], [], [], [], []], "Ev Idx": 10375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700879.862, "dur": 0.414,
    "args": {
      "External id": 2482673,"Record function id": 0, "Concrete Inputs": ["", "[8192]", "[1]", "57344"], "Input type": ["float", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656700939.460, "dur": 83.013,
    "args": {
      "External id": 2482674,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[32000, 1], [2048, 1]], "Input Dims": [[8192, 32000], [32000, 2048]], "Ev Idx": 10377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::slice", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701029.754, "dur": 6.224,
    "args": {
      "External id": 2482675,"Record function id": 0, "Concrete Inputs": ["", "0", "57344", "65536", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar", "Scalar", "Scalar"], "Input Strides": [[2048, 1], [], [], [], []], "Input Dims": [[65536, 2048], [], [], [], []], "Ev Idx": 10378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701033.293, "dur": 1.046,
    "args": {
      "External id": 2482676,"Record function id": 0, "Concrete Inputs": ["", "[8192, 2048]", "[2048, 1]", "117440512"], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", "Scalar"], "Input Strides": [[2048, 1], [], [], []], "Input Dims": [[65536, 2048], [], [], []], "Ev Idx": 10379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701037.375, "dur": 27.659,
    "args": {
      "External id": 2482677,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[8192, 2048], [8192, 2048], []], "Ev Idx": 10380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::t", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701070.689, "dur": 7.404,
    "args": {
      "External id": 2482678,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[32000, 1]], "Input Dims": [[8192, 32000]], "Ev Idx": 10381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::transpose", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701072.379, "dur": 5.036,
    "args": {
      "External id": 2482679,"Record function id": 0, "Concrete Inputs": ["", "0", "1"], "Input type": ["c10::BFloat16", "Scalar", "Scalar"], "Input Strides": [[32000, 1], [], []], "Input Dims": [[8192, 32000], [], []], "Ev Idx": 10382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701075.693, "dur": 1.540,
    "args": {
      "External id": 2482680,"Record function id": 0, "Concrete Inputs": ["", "[32000, 8192]", "[1, 32000]", ""], "Input type": ["c10::BFloat16", "ScalarList", "ScalarList", ""], "Input Strides": [[32000, 1], [], [], []], "Input Dims": [[8192, 32000], [], [], []], "Ev Idx": 10383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::matmul", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701081.233, "dur": 43.902,
    "args": {
      "External id": 2482681,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mm", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701082.486, "dur": 41.946,
    "args": {
      "External id": 2482682,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["c10::BFloat16", "c10::BFloat16"], "Input Strides": [[1, 32000], [2048, 1]], "Input Dims": [[32000, 8192], [8192, 2048]], "Ev Idx": 10385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701129.367, "dur": 13.793,
    "args": {
      "External id": 2482683,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "c10::BFloat16", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701147.955, "dur": 24.656,
    "args": {
      "External id": 2482684,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[1], []], "Input Dims": [[65536], []], "Ev Idx": 10387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::sum", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701150.279, "dur": 21.909,
    "args": {
      "External id": 2482685,"Record function id": 0, "Concrete Inputs": ["", "[]", "False", ""], "Input type": ["float", "ScalarList", "Scalar", ""], "Input Strides": [[1], [], [], []], "Input Dims": [[65536], [], [], []], "Ev Idx": 10388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701154.869, "dur": 2.872,
    "args": {
      "External id": 2482686,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701178.141, "dur": 68.540,
    "args": {
      "External id": 2482687,"Record function id": 0, "Concrete Inputs": ["", "", "15", "False", "False", ""], "Input type": ["float", "", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], []], "Ev Idx": 10390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_to_copy", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701179.771, "dur": 66.499,
    "args": {
      "External id": 2482688,"Record function id": 0, "Concrete Inputs": ["", "15", "", "", "", "False", ""], "Input type": ["float", "Scalar", "", "", "", "Scalar", ""], "Input Strides": [[2048, 1], [], [], [], [], [], []], "Input Dims": [[32000, 2048], [], [], [], [], [], []], "Ev Idx": 10391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701226.214, "dur": 5.247,
    "args": {
      "External id": 2482689,"Record function id": 0, "Concrete Inputs": ["[32000, 2048]", "[2048, 1]", "15", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701232.510, "dur": 13.163,
    "args": {
      "External id": 2482690,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["c10::BFloat16", "float", "Scalar"], "Input Strides": [[2048, 1], [2048, 1], []], "Input Dims": [[32000, 2048], [32000, 2048], []], "Ev Idx": 10393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701259.711, "dur": 4.850,
    "args": {
      "External id": 2482691,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 10394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701261.380, "dur": 2.921,
    "args": {
      "External id": 2482692,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[65536, 2048]], "Ev Idx": 10395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701265.514, "dur": 0.977,
    "args": {
      "External id": 2482693,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "detach", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701265.920, "dur": 0.488,
    "args": {
      "External id": 2482694,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["c10::BFloat16"], "Input Strides": [[2048, 1]], "Input Dims": [[32000, 2048]], "Ev Idx": 10397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701308.627, "dur": 23.338,
    "args": {
      "External id": 2482695,"Sequence number": 24740648, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701333.838, "dur": 12.953,
    "args": {
      "External id": 2482696,"Sequence number": 24740649, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10399
    }
  },
  {
    "ph": "s", "id": 2, "pid": 1336757, "tid": 1336757, "ts": 1514656701333.838,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "FSDP::post_forward", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701452.655, "dur": 38.608,
    "args": {
      "External id": 2482697,"Record function id": 0, "Ev Idx": 10400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701617.465, "dur": 36.853,
    "args": {
      "External id": 2482698,"Sequence number": 24740650, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 10401
    }
  },
  {
    "ph": "s", "id": 1, "pid": 1336757, "tid": 1336757, "ts": 1514656701617.465,
    "cat": "fwdbwd", "name": "fwdbwd"
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ones_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701712.620, "dur": 26.167,
    "args": {
      "External id": 2482699,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "1"], "Input type": ["float", "", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[1], [], [], [], [], []], "Ev Idx": 10402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701714.260, "dur": 8.092,
    "args": {
      "External id": 2482700,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "False", "1"], "Input type": ["float", "", "", "", "Scalar", "Scalar"], "Input Strides": [[1], [], [], [], [], []], "Input Dims": [[1], [], [], [], [], []], "Ev Idx": 10403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701717.675, "dur": 3.975,
    "args": {
      "External id": 2482701,"Record function id": 0, "Concrete Inputs": ["[1]", "[1]", "6", "0", "", "False"], "Input type": ["ScalarList", "ScalarList", "Scalar", "Scalar", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514656701723.829, "dur": 14.517,
    "args": {
      "External id": 2482702,"Record function id": 0, "Concrete Inputs": ["", "1."], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 10405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336757, "tid": 1336757,
    "ts": 1514658027435.572, "dur": 90.775,
    "args": {
      "External id": 2482703,"Sequence number": 24740651, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::div", "pid": 1336757, "tid": 1336757,
    "ts": 1514658027557.174, "dur": 28.475,
    "args": {
      "External id": 2482704,"Sequence number": 24740652, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "long int"], "Input Strides": [[1], []], "Input Dims": [[1], []], "Ev Idx": 10407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1336757,
    "ts": 1514658027595.051, "dur": 23.343,
    "args": {
      "External id": 2482705,"Sequence number": 24740653, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "long int", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1336757,
    "ts": 1514658028207.001, "dur": 30.558,
    "args": {
      "External id": 2482706,"Sequence number": 24740654, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "long int", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 10409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1336757,
    "ts": 1514658028816.723, "dur": 33.686,
    "args": {
      "External id": 2482707,"Sequence number": 24740655, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "long int", "Scalar"], "Input Strides": [[1], [], []], "Input Dims": [[1], [], []], "Ev Idx": 10410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_norm", "pid": 1336757, "tid": 1336757,
    "ts": 1514658030570.321, "dur": 3157.003,
    "args": {
      "External id": 2482708,"Record function id": 0, "Concrete Inputs": ["", "2.", ""], "Input type": ["TensorList", "Scalar", ""], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 10411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_norm", "pid": 1336757, "tid": 1336757,
    "ts": 1514658031226.846, "dur": 1013.849,
    "args": {
      "External id": 2482709,"Record function id": 0, "Concrete Inputs": ["", "2.", ""], "Input type": ["TensorList", "Scalar", ""], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 10412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zeros", "pid": 1336757, "tid": 1336757,
    "ts": 1514658031247.025, "dur": 68.376,
    "args": {
      "External id": 2482710,"Record function id": 0, "Concrete Inputs": ["[36375]", "6", "0", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 10413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514658031250.525, "dur": 14.064,
    "args": {
      "External id": 2482711,"Record function id": 0, "Concrete Inputs": ["[36375]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 10414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::zero_", "pid": 1336757, "tid": 1336757,
    "ts": 1514658031267.644, "dur": 47.462,
    "args": {
      "External id": 2482712,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[1]], "Input Dims": [[36375]], "Ev Idx": 10415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::fill_", "pid": 1336757, "tid": 1336757,
    "ts": 1514658031269.619, "dur": 44.912,
    "args": {
      "External id": 2482713,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[1], []], "Input Dims": [[36375], []], "Ev Idx": 10416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033762.483, "dur": 3.792,
    "args": {
      "External id": 2482714,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033768.543, "dur": 0.434,
    "args": {
      "External id": 2482715,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033770.240, "dur": 0.417,
    "args": {
      "External id": 2482716,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033772.021, "dur": 0.258,
    "args": {
      "External id": 2482717,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033773.174, "dur": 0.239,
    "args": {
      "External id": 2482718,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033774.662, "dur": 0.235,
    "args": {
      "External id": 2482719,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033775.904, "dur": 0.307,
    "args": {
      "External id": 2482720,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033778.847, "dur": 0.322,
    "args": {
      "External id": 2482721,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033780.034, "dur": 0.392,
    "args": {
      "External id": 2482722,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033781.588, "dur": 0.353,
    "args": {
      "External id": 2482723,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033782.747, "dur": 0.335,
    "args": {
      "External id": 2482724,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033784.147, "dur": 0.384,
    "args": {
      "External id": 2482725,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033785.311, "dur": 0.461,
    "args": {
      "External id": 2482726,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033786.734, "dur": 0.457,
    "args": {
      "External id": 2482727,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033788.162, "dur": 0.400,
    "args": {
      "External id": 2482728,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033790.969, "dur": 0.337,
    "args": {
      "External id": 2482729,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033792.217, "dur": 0.367,
    "args": {
      "External id": 2482730,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033793.545, "dur": 0.378,
    "args": {
      "External id": 2482731,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033795.025, "dur": 0.448,
    "args": {
      "External id": 2482732,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033796.421, "dur": 0.392,
    "args": {
      "External id": 2482733,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033797.630, "dur": 0.355,
    "args": {
      "External id": 2482734,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033799.051, "dur": 0.352,
    "args": {
      "External id": 2482735,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033800.290, "dur": 0.368,
    "args": {
      "External id": 2482736,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033803.237, "dur": 0.296,
    "args": {
      "External id": 2482737,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033804.511, "dur": 0.207,
    "args": {
      "External id": 2482738,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033805.893, "dur": 0.212,
    "args": {
      "External id": 2482739,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033807.039, "dur": 0.200,
    "args": {
      "External id": 2482740,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033808.298, "dur": 0.208,
    "args": {
      "External id": 2482741,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033809.401, "dur": 0.220,
    "args": {
      "External id": 2482742,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033810.581, "dur": 0.227,
    "args": {
      "External id": 2482743,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033811.694, "dur": 0.239,
    "args": {
      "External id": 2482744,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033814.358, "dur": 0.203,
    "args": {
      "External id": 2482745,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033815.412, "dur": 0.291,
    "args": {
      "External id": 2482746,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033816.815, "dur": 0.349,
    "args": {
      "External id": 2482747,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033817.946, "dur": 0.355,
    "args": {
      "External id": 2482748,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033819.073, "dur": 0.198,
    "args": {
      "External id": 2482749,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033820.065, "dur": 0.306,
    "args": {
      "External id": 2482750,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033821.153, "dur": 0.334,
    "args": {
      "External id": 2482751,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033822.267, "dur": 0.297,
    "args": {
      "External id": 2482752,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033824.903, "dur": 0.345,
    "args": {
      "External id": 2482753,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033826.134, "dur": 0.307,
    "args": {
      "External id": 2482754,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033827.471, "dur": 0.337,
    "args": {
      "External id": 2482755,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033828.599, "dur": 0.206,
    "args": {
      "External id": 2482756,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033829.638, "dur": 0.205,
    "args": {
      "External id": 2482757,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033830.623, "dur": 0.196,
    "args": {
      "External id": 2482758,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033833.059, "dur": 0.222,
    "args": {
      "External id": 2482759,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033834.132, "dur": 0.208,
    "args": {
      "External id": 2482760,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033836.432, "dur": 0.406,
    "args": {
      "External id": 2482761,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033837.725, "dur": 0.347,
    "args": {
      "External id": 2482762,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033838.828, "dur": 0.363,
    "args": {
      "External id": 2482763,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033840.044, "dur": 0.358,
    "args": {
      "External id": 2482764,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033841.303, "dur": 0.202,
    "args": {
      "External id": 2482765,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033842.259, "dur": 0.334,
    "args": {
      "External id": 2482766,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033843.564, "dur": 0.338,
    "args": {
      "External id": 2482767,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033844.663, "dur": 0.307,
    "args": {
      "External id": 2482768,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033847.433, "dur": 0.355,
    "args": {
      "External id": 2482769,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033848.617, "dur": 0.201,
    "args": {
      "External id": 2482770,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033849.592, "dur": 0.203,
    "args": {
      "External id": 2482771,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033850.560, "dur": 0.201,
    "args": {
      "External id": 2482772,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033851.552, "dur": 0.196,
    "args": {
      "External id": 2482773,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033852.629, "dur": 0.225,
    "args": {
      "External id": 2482774,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033853.706, "dur": 0.205,
    "args": {
      "External id": 2482775,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033854.809, "dur": 0.201,
    "args": {
      "External id": 2482776,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033857.472, "dur": 0.202,
    "args": {
      "External id": 2482777,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033858.520, "dur": 0.199,
    "args": {
      "External id": 2482778,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033859.801, "dur": 0.202,
    "args": {
      "External id": 2482779,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033860.870, "dur": 0.193,
    "args": {
      "External id": 2482780,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033861.845, "dur": 0.206,
    "args": {
      "External id": 2482781,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033862.921, "dur": 0.204,
    "args": {
      "External id": 2482782,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033863.964, "dur": 0.200,
    "args": {
      "External id": 2482783,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033865.012, "dur": 0.199,
    "args": {
      "External id": 2482784,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033867.825, "dur": 0.199,
    "args": {
      "External id": 2482785,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033868.851, "dur": 0.197,
    "args": {
      "External id": 2482786,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033869.812, "dur": 0.219,
    "args": {
      "External id": 2482787,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033870.768, "dur": 0.195,
    "args": {
      "External id": 2482788,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033871.708, "dur": 0.197,
    "args": {
      "External id": 2482789,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033872.637, "dur": 0.197,
    "args": {
      "External id": 2482790,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033874.482, "dur": 0.202,
    "args": {
      "External id": 2482791,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033875.450, "dur": 0.198,
    "args": {
      "External id": 2482792,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033878.318, "dur": 0.200,
    "args": {
      "External id": 2482793,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033879.329, "dur": 0.203,
    "args": {
      "External id": 2482794,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033880.360, "dur": 0.194,
    "args": {
      "External id": 2482795,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033881.445, "dur": 0.203,
    "args": {
      "External id": 2482796,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033882.412, "dur": 0.201,
    "args": {
      "External id": 2482797,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033883.522, "dur": 0.197,
    "args": {
      "External id": 2482798,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033884.512, "dur": 0.197,
    "args": {
      "External id": 2482799,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033885.664, "dur": 0.203,
    "args": {
      "External id": 2482800,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033887.781, "dur": 0.197,
    "args": {
      "External id": 2482801,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033888.762, "dur": 0.322,
    "args": {
      "External id": 2482802,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033889.941, "dur": 0.216,
    "args": {
      "External id": 2482803,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033890.938, "dur": 0.192,
    "args": {
      "External id": 2482804,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033891.953, "dur": 0.201,
    "args": {
      "External id": 2482805,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033892.982, "dur": 0.333,
    "args": {
      "External id": 2482806,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033894.848, "dur": 0.195,
    "args": {
      "External id": 2482807,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033895.868, "dur": 0.222,
    "args": {
      "External id": 2482808,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033898.543, "dur": 0.288,
    "args": {
      "External id": 2482809,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033899.676, "dur": 0.196,
    "args": {
      "External id": 2482810,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033900.657, "dur": 0.203,
    "args": {
      "External id": 2482811,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033901.652, "dur": 0.201,
    "args": {
      "External id": 2482812,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033902.616, "dur": 0.192,
    "args": {
      "External id": 2482813,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033903.588, "dur": 0.201,
    "args": {
      "External id": 2482814,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033904.535, "dur": 0.223,
    "args": {
      "External id": 2482815,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033905.531, "dur": 0.317,
    "args": {
      "External id": 2482816,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033908.022, "dur": 0.263,
    "args": {
      "External id": 2482817,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033909.036, "dur": 0.319,
    "args": {
      "External id": 2482818,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033910.125, "dur": 0.339,
    "args": {
      "External id": 2482819,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033911.581, "dur": 0.333,
    "args": {
      "External id": 2482820,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033912.756, "dur": 0.203,
    "args": {
      "External id": 2482821,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033913.854, "dur": 0.317,
    "args": {
      "External id": 2482822,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033915.476, "dur": 0.306,
    "args": {
      "External id": 2482823,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033916.657, "dur": 0.297,
    "args": {
      "External id": 2482824,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033919.250, "dur": 0.344,
    "args": {
      "External id": 2482825,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033920.471, "dur": 0.197,
    "args": {
      "External id": 2482826,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033921.417, "dur": 0.198,
    "args": {
      "External id": 2482827,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033922.481, "dur": 0.226,
    "args": {
      "External id": 2482828,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033923.598, "dur": 0.220,
    "args": {
      "External id": 2482829,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033924.693, "dur": 0.198,
    "args": {
      "External id": 2482830,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033925.647, "dur": 0.195,
    "args": {
      "External id": 2482831,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033926.647, "dur": 0.200,
    "args": {
      "External id": 2482832,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033929.175, "dur": 0.197,
    "args": {
      "External id": 2482833,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033930.158, "dur": 0.315,
    "args": {
      "External id": 2482834,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033931.260, "dur": 0.338,
    "args": {
      "External id": 2482835,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033932.340, "dur": 0.329,
    "args": {
      "External id": 2482836,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033933.424, "dur": 0.193,
    "args": {
      "External id": 2482837,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033934.406, "dur": 0.281,
    "args": {
      "External id": 2482838,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033935.443, "dur": 0.296,
    "args": {
      "External id": 2482839,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033936.506, "dur": 0.320,
    "args": {
      "External id": 2482840,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033939.168, "dur": 0.305,
    "args": {
      "External id": 2482841,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033940.236, "dur": 0.323,
    "args": {
      "External id": 2482842,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033941.925, "dur": 0.197,
    "args": {
      "External id": 2482843,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033942.912, "dur": 0.196,
    "args": {
      "External id": 2482844,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033943.956, "dur": 0.199,
    "args": {
      "External id": 2482845,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033944.912, "dur": 0.198,
    "args": {
      "External id": 2482846,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033945.899, "dur": 0.199,
    "args": {
      "External id": 2482847,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033946.838, "dur": 0.197,
    "args": {
      "External id": 2482848,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033949.507, "dur": 0.209,
    "args": {
      "External id": 2482849,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033950.564, "dur": 0.196,
    "args": {
      "External id": 2482850,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033951.525, "dur": 0.200,
    "args": {
      "External id": 2482851,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033952.527, "dur": 0.195,
    "args": {
      "External id": 2482852,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033953.565, "dur": 0.197,
    "args": {
      "External id": 2482853,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033954.534, "dur": 0.198,
    "args": {
      "External id": 2482854,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033955.498, "dur": 0.193,
    "args": {
      "External id": 2482855,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033956.638, "dur": 0.201,
    "args": {
      "External id": 2482856,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033959.132, "dur": 0.201,
    "args": {
      "External id": 2482857,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033960.217, "dur": 0.223,
    "args": {
      "External id": 2482858,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033961.198, "dur": 0.218,
    "args": {
      "External id": 2482859,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033962.272, "dur": 0.200,
    "args": {
      "External id": 2482860,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033963.212, "dur": 0.196,
    "args": {
      "External id": 2482861,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033964.210, "dur": 0.193,
    "args": {
      "External id": 2482862,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033965.902, "dur": 0.201,
    "args": {
      "External id": 2482863,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033966.985, "dur": 0.200,
    "args": {
      "External id": 2482864,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033969.507, "dur": 0.191,
    "args": {
      "External id": 2482865,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033970.578, "dur": 0.209,
    "args": {
      "External id": 2482866,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033971.591, "dur": 0.196,
    "args": {
      "External id": 2482867,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033972.767, "dur": 0.197,
    "args": {
      "External id": 2482868,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033973.824, "dur": 0.200,
    "args": {
      "External id": 2482869,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033974.874, "dur": 0.197,
    "args": {
      "External id": 2482870,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033975.924, "dur": 0.199,
    "args": {
      "External id": 2482871,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033976.878, "dur": 0.229,
    "args": {
      "External id": 2482872,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033980.410, "dur": 0.219,
    "args": {
      "External id": 2482873,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658033981.396, "dur": 0.195,
    "args": {
      "External id": 2482874,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034008.151, "dur": 0.979,
    "args": {
      "External id": 2482875,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034012.953, "dur": 0.375,
    "args": {
      "External id": 2482876,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034014.248, "dur": 0.347,
    "args": {
      "External id": 2482877,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034015.502, "dur": 0.367,
    "args": {
      "External id": 2482878,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034016.731, "dur": 0.395,
    "args": {
      "External id": 2482879,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034018.069, "dur": 0.346,
    "args": {
      "External id": 2482880,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034020.786, "dur": 0.208,
    "args": {
      "External id": 2482881,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034021.857, "dur": 0.220,
    "args": {
      "External id": 2482882,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034023.038, "dur": 0.203,
    "args": {
      "External id": 2482883,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034024.097, "dur": 0.209,
    "args": {
      "External id": 2482884,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034025.254, "dur": 0.206,
    "args": {
      "External id": 2482885,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034026.206, "dur": 0.202,
    "args": {
      "External id": 2482886,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034027.638, "dur": 0.209,
    "args": {
      "External id": 2482887,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034028.791, "dur": 0.207,
    "args": {
      "External id": 2482888,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034031.183, "dur": 0.396,
    "args": {
      "External id": 2482889,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034032.364, "dur": 0.416,
    "args": {
      "External id": 2482890,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034033.537, "dur": 0.316,
    "args": {
      "External id": 2482891,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034034.609, "dur": 0.251,
    "args": {
      "External id": 2482892,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034035.604, "dur": 0.211,
    "args": {
      "External id": 2482893,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034036.611, "dur": 0.324,
    "args": {
      "External id": 2482894,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034037.721, "dur": 0.319,
    "args": {
      "External id": 2482895,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034038.857, "dur": 0.320,
    "args": {
      "External id": 2482896,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034041.561, "dur": 0.300,
    "args": {
      "External id": 2482897,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034042.667, "dur": 0.322,
    "args": {
      "External id": 2482898,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034043.750, "dur": 0.363,
    "args": {
      "External id": 2482899,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034044.874, "dur": 0.200,
    "args": {
      "External id": 2482900,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034045.874, "dur": 0.203,
    "args": {
      "External id": 2482901,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034046.811, "dur": 0.210,
    "args": {
      "External id": 2482902,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034047.777, "dur": 0.200,
    "args": {
      "External id": 2482903,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034048.740, "dur": 0.202,
    "args": {
      "External id": 2482904,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034051.111, "dur": 0.210,
    "args": {
      "External id": 2482905,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034052.091, "dur": 0.225,
    "args": {
      "External id": 2482906,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034053.067, "dur": 0.434,
    "args": {
      "External id": 2482907,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034054.260, "dur": 0.337,
    "args": {
      "External id": 2482908,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034055.388, "dur": 0.207,
    "args": {
      "External id": 2482909,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034056.494, "dur": 0.303,
    "args": {
      "External id": 2482910,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034057.680, "dur": 0.200,
    "args": {
      "External id": 2482911,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034058.768, "dur": 0.346,
    "args": {
      "External id": 2482912,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034061.407, "dur": 0.326,
    "args": {
      "External id": 2482913,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034062.720, "dur": 0.339,
    "args": {
      "External id": 2482914,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034064.291, "dur": 0.352,
    "args": {
      "External id": 2482915,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034065.439, "dur": 0.198,
    "args": {
      "External id": 2482916,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034066.386, "dur": 0.205,
    "args": {
      "External id": 2482917,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034067.355, "dur": 0.211,
    "args": {
      "External id": 2482918,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034068.393, "dur": 0.222,
    "args": {
      "External id": 2482919,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034069.366, "dur": 0.204,
    "args": {
      "External id": 2482920,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034071.553, "dur": 0.209,
    "args": {
      "External id": 2482921,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034072.557, "dur": 0.199,
    "args": {
      "External id": 2482922,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034073.513, "dur": 0.199,
    "args": {
      "External id": 2482923,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034074.510, "dur": 0.202,
    "args": {
      "External id": 2482924,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034075.492, "dur": 0.207,
    "args": {
      "External id": 2482925,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034076.517, "dur": 0.201,
    "args": {
      "External id": 2482926,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034077.475, "dur": 0.204,
    "args": {
      "External id": 2482927,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034078.419, "dur": 0.212,
    "args": {
      "External id": 2482928,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034080.931, "dur": 0.201,
    "args": {
      "External id": 2482929,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034081.888, "dur": 0.204,
    "args": {
      "External id": 2482930,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034082.832, "dur": 0.207,
    "args": {
      "External id": 2482931,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10634
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034083.811, "dur": 0.200,
    "args": {
      "External id": 2482932,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10635
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034084.770, "dur": 0.321,
    "args": {
      "External id": 2482933,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10636
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034085.855, "dur": 0.331,
    "args": {
      "External id": 2482934,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10637
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034086.992, "dur": 0.340,
    "args": {
      "External id": 2482935,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10638
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034088.161, "dur": 0.327,
    "args": {
      "External id": 2482936,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10639
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034090.638, "dur": 0.213,
    "args": {
      "External id": 2482937,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10640
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034091.692, "dur": 0.210,
    "args": {
      "External id": 2482938,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10641
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034092.645, "dur": 0.199,
    "args": {
      "External id": 2482939,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10642
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034093.666, "dur": 0.219,
    "args": {
      "External id": 2482940,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10643
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034094.707, "dur": 0.211,
    "args": {
      "External id": 2482941,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10644
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034095.672, "dur": 0.200,
    "args": {
      "External id": 2482942,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10645
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034096.736, "dur": 0.215,
    "args": {
      "External id": 2482943,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10646
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034097.786, "dur": 0.214,
    "args": {
      "External id": 2482944,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10647
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034100.807, "dur": 0.201,
    "args": {
      "External id": 2482945,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10648
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034101.764, "dur": 0.206,
    "args": {
      "External id": 2482946,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10649
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034103.041, "dur": 0.209,
    "args": {
      "External id": 2482947,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10650
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034104.231, "dur": 0.209,
    "args": {
      "External id": 2482948,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10651
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034105.345, "dur": 0.214,
    "args": {
      "External id": 2482949,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10652
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034106.338, "dur": 0.246,
    "args": {
      "External id": 2482950,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10653
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034107.339, "dur": 0.345,
    "args": {
      "External id": 2482951,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10654
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034108.439, "dur": 0.332,
    "args": {
      "External id": 2482952,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10655
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034111.219, "dur": 0.205,
    "args": {
      "External id": 2482953,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10656
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034112.338, "dur": 0.200,
    "args": {
      "External id": 2482954,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10657
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034113.384, "dur": 0.200,
    "args": {
      "External id": 2482955,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10658
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034114.457, "dur": 0.200,
    "args": {
      "External id": 2482956,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10659
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034115.551, "dur": 0.208,
    "args": {
      "External id": 2482957,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10660
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034116.510, "dur": 0.201,
    "args": {
      "External id": 2482958,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10661
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034117.651, "dur": 0.204,
    "args": {
      "External id": 2482959,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10662
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034118.691, "dur": 0.208,
    "args": {
      "External id": 2482960,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10663
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034121.284, "dur": 0.200,
    "args": {
      "External id": 2482961,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10664
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034122.248, "dur": 0.201,
    "args": {
      "External id": 2482962,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10665
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034123.179, "dur": 0.418,
    "args": {
      "External id": 2482963,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10666
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034124.425, "dur": 0.356,
    "args": {
      "External id": 2482964,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10667
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034125.620, "dur": 0.212,
    "args": {
      "External id": 2482965,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10668
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034126.706, "dur": 0.312,
    "args": {
      "External id": 2482966,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10669
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034127.844, "dur": 0.297,
    "args": {
      "External id": 2482967,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10670
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034128.990, "dur": 0.276,
    "args": {
      "External id": 2482968,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10671
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034131.441, "dur": 0.317,
    "args": {
      "External id": 2482969,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10672
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034135.669, "dur": 0.286,
    "args": {
      "External id": 2482970,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10673
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034136.823, "dur": 0.320,
    "args": {
      "External id": 2482971,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10674
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034138.013, "dur": 0.344,
    "args": {
      "External id": 2482972,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10675
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034139.123, "dur": 0.203,
    "args": {
      "External id": 2482973,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10676
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034140.171, "dur": 0.344,
    "args": {
      "External id": 2482974,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10677
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034141.311, "dur": 0.199,
    "args": {
      "External id": 2482975,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10678
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034142.289, "dur": 0.205,
    "args": {
      "External id": 2482976,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10679
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034145.250, "dur": 0.209,
    "args": {
      "External id": 2482977,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10680
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034146.318, "dur": 0.203,
    "args": {
      "External id": 2482978,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10681
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034147.324, "dur": 0.200,
    "args": {
      "External id": 2482979,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10682
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034148.299, "dur": 0.393,
    "args": {
      "External id": 2482980,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10683
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034149.468, "dur": 0.200,
    "args": {
      "External id": 2482981,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10684
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034150.479, "dur": 0.337,
    "args": {
      "External id": 2482982,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10685
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034152.094, "dur": 0.325,
    "args": {
      "External id": 2482983,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10686
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034153.174, "dur": 0.336,
    "args": {
      "External id": 2482984,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10687
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034155.761, "dur": 0.360,
    "args": {
      "External id": 2482985,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10688
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034157.002, "dur": 0.360,
    "args": {
      "External id": 2482986,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10689
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034159.626, "dur": 0.403,
    "args": {
      "External id": 2482987,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10690
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034160.988, "dur": 0.340,
    "args": {
      "External id": 2482988,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10691
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034162.215, "dur": 0.353,
    "args": {
      "External id": 2482989,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10692
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034163.409, "dur": 0.328,
    "args": {
      "External id": 2482990,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10693
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034164.644, "dur": 0.331,
    "args": {
      "External id": 2482991,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10694
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034166.020, "dur": 0.359,
    "args": {
      "External id": 2482992,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10695
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034168.643, "dur": 0.200,
    "args": {
      "External id": 2482993,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10696
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034169.681, "dur": 0.210,
    "args": {
      "External id": 2482994,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10697
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034170.632, "dur": 0.203,
    "args": {
      "External id": 2482995,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10698
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034171.743, "dur": 0.227,
    "args": {
      "External id": 2482996,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10699
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034172.826, "dur": 0.202,
    "args": {
      "External id": 2482997,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10700
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034173.888, "dur": 0.202,
    "args": {
      "External id": 2482998,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10701
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034175.121, "dur": 0.195,
    "args": {
      "External id": 2482999,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10702
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034176.210, "dur": 0.212,
    "args": {
      "External id": 2483000,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10703
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034178.497, "dur": 0.199,
    "args": {
      "External id": 2483001,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10704
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034179.451, "dur": 0.196,
    "args": {
      "External id": 2483002,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10705
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034180.419, "dur": 0.197,
    "args": {
      "External id": 2483003,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10706
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034181.352, "dur": 0.225,
    "args": {
      "External id": 2483004,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 10707
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::stack", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034238.640, "dur": 1532.796,
    "args": {
      "External id": 2483005,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10708
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::stack", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034691.841, "dur": 1005.310,
    "args": {
      "External id": 2483006,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10709
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034701.262, "dur": 7.997,
    "args": {
      "External id": 2483007,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10710
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034705.510, "dur": 3.151,
    "args": {
      "External id": 2483008,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10711
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034709.937, "dur": 3.053,
    "args": {
      "External id": 2483009,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10712
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034710.742, "dur": 2.121,
    "args": {
      "External id": 2483010,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10713
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034713.538, "dur": 3.002,
    "args": {
      "External id": 2483011,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10714
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034715.833, "dur": 0.632,
    "args": {
      "External id": 2483012,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10715
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034717.080, "dur": 1.330,
    "args": {
      "External id": 2483013,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10716
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034717.658, "dur": 0.679,
    "args": {
      "External id": 2483014,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10717
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034720.429, "dur": 4.318,
    "args": {
      "External id": 2483015,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10718
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034724.142, "dur": 0.539,
    "args": {
      "External id": 2483016,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10719
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034725.232, "dur": 1.259,
    "args": {
      "External id": 2483017,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10720
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034725.788, "dur": 0.641,
    "args": {
      "External id": 2483018,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10721
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034726.887, "dur": 1.362,
    "args": {
      "External id": 2483019,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10722
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034727.445, "dur": 0.732,
    "args": {
      "External id": 2483020,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10723
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034728.807, "dur": 3.658,
    "args": {
      "External id": 2483021,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10724
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034731.418, "dur": 0.979,
    "args": {
      "External id": 2483022,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10725
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034732.791, "dur": 6.092,
    "args": {
      "External id": 2483023,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10726
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034733.297, "dur": 5.415,
    "args": {
      "External id": 2483024,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10727
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034739.160, "dur": 2.953,
    "args": {
      "External id": 2483025,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10728
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034739.520, "dur": 2.517,
    "args": {
      "External id": 2483026,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10729
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034742.351, "dur": 2.426,
    "args": {
      "External id": 2483027,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10730
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034743.900, "dur": 0.809,
    "args": {
      "External id": 2483028,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10731
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034745.042, "dur": 1.338,
    "args": {
      "External id": 2483029,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10732
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034745.560, "dur": 0.745,
    "args": {
      "External id": 2483030,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10733
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034748.377, "dur": 4.177,
    "args": {
      "External id": 2483031,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10734
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034751.775, "dur": 0.711,
    "args": {
      "External id": 2483032,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10735
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034753.015, "dur": 1.438,
    "args": {
      "External id": 2483033,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10736
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034753.518, "dur": 0.716,
    "args": {
      "External id": 2483034,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10737
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034754.811, "dur": 1.389,
    "args": {
      "External id": 2483035,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10738
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034755.468, "dur": 0.657,
    "args": {
      "External id": 2483036,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10739
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034756.463, "dur": 3.519,
    "args": {
      "External id": 2483037,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10740
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034759.367, "dur": 0.549,
    "args": {
      "External id": 2483038,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10741
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034760.399, "dur": 1.372,
    "args": {
      "External id": 2483039,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10742
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034761.065, "dur": 0.642,
    "args": {
      "External id": 2483040,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10743
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034762.119, "dur": 2.276,
    "args": {
      "External id": 2483041,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10744
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034762.584, "dur": 1.568,
    "args": {
      "External id": 2483042,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10745
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034764.759, "dur": 2.313,
    "args": {
      "External id": 2483043,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10746
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034766.474, "dur": 0.531,
    "args": {
      "External id": 2483044,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10747
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034767.303, "dur": 1.214,
    "args": {
      "External id": 2483045,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10748
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034767.738, "dur": 0.708,
    "args": {
      "External id": 2483046,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10749
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034770.442, "dur": 4.069,
    "args": {
      "External id": 2483047,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10750
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034773.815, "dur": 0.601,
    "args": {
      "External id": 2483048,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10751
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034774.798, "dur": 0.987,
    "args": {
      "External id": 2483049,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10752
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034775.157, "dur": 0.535,
    "args": {
      "External id": 2483050,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10753
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034776.032, "dur": 1.146,
    "args": {
      "External id": 2483051,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10754
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034776.508, "dur": 0.598,
    "args": {
      "External id": 2483052,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10755
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034777.408, "dur": 3.284,
    "args": {
      "External id": 2483053,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10756
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034779.673, "dur": 0.653,
    "args": {
      "External id": 2483054,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10757
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034781.012, "dur": 1.273,
    "args": {
      "External id": 2483055,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10758
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034781.495, "dur": 0.637,
    "args": {
      "External id": 2483056,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10759
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034782.611, "dur": 2.678,
    "args": {
      "External id": 2483057,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10760
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034783.096, "dur": 1.989,
    "args": {
      "External id": 2483058,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10761
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034785.742, "dur": 2.816,
    "args": {
      "External id": 2483059,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10762
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034787.682, "dur": 0.810,
    "args": {
      "External id": 2483060,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10763
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034788.892, "dur": 1.025,
    "args": {
      "External id": 2483061,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10764
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034789.342, "dur": 0.502,
    "args": {
      "External id": 2483062,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10765
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034791.527, "dur": 4.165,
    "args": {
      "External id": 2483063,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10766
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034794.918, "dur": 0.712,
    "args": {
      "External id": 2483064,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10767
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034795.940, "dur": 1.230,
    "args": {
      "External id": 2483065,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10768
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034796.393, "dur": 0.705,
    "args": {
      "External id": 2483066,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10769
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034797.397, "dur": 1.291,
    "args": {
      "External id": 2483067,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10770
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034797.765, "dur": 0.854,
    "args": {
      "External id": 2483068,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10771
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034798.937, "dur": 3.130,
    "args": {
      "External id": 2483069,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10772
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034801.177, "dur": 0.803,
    "args": {
      "External id": 2483070,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10773
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034802.469, "dur": 1.373,
    "args": {
      "External id": 2483071,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10774
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034802.986, "dur": 0.766,
    "args": {
      "External id": 2483072,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10775
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034804.222, "dur": 2.713,
    "args": {
      "External id": 2483073,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10776
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034804.763, "dur": 1.964,
    "args": {
      "External id": 2483074,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10777
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034807.291, "dur": 3.637,
    "args": {
      "External id": 2483075,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10778
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034810.155, "dur": 0.542,
    "args": {
      "External id": 2483076,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10779
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034811.317, "dur": 2.169,
    "args": {
      "External id": 2483077,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10780
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034811.680, "dur": 1.729,
    "args": {
      "External id": 2483078,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10781
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034815.230, "dur": 2.470,
    "args": {
      "External id": 2483079,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10782
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034816.910, "dur": 0.722,
    "args": {
      "External id": 2483080,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10783
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034818.055, "dur": 1.347,
    "args": {
      "External id": 2483081,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10784
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034818.696, "dur": 0.632,
    "args": {
      "External id": 2483082,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10785
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034819.767, "dur": 2.801,
    "args": {
      "External id": 2483083,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10786
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034821.916, "dur": 0.558,
    "args": {
      "External id": 2483084,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10787
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034822.976, "dur": 2.853,
    "args": {
      "External id": 2483085,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10788
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034824.758, "dur": 0.838,
    "args": {
      "External id": 2483086,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10789
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034826.156, "dur": 1.361,
    "args": {
      "External id": 2483087,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10790
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034826.679, "dur": 0.766,
    "args": {
      "External id": 2483088,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10791
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034827.753, "dur": 2.987,
    "args": {
      "External id": 2483089,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10792
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034829.980, "dur": 0.694,
    "args": {
      "External id": 2483090,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10793
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034830.970, "dur": 2.381,
    "args": {
      "External id": 2483091,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10794
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034832.609, "dur": 0.648,
    "args": {
      "External id": 2483092,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10795
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034833.587, "dur": 2.925,
    "args": {
      "External id": 2483093,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10796
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034834.273, "dur": 2.162,
    "args": {
      "External id": 2483094,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10797
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034838.175, "dur": 2.067,
    "args": {
      "External id": 2483095,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10798
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034839.521, "dur": 0.655,
    "args": {
      "External id": 2483096,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10799
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034840.596, "dur": 1.294,
    "args": {
      "External id": 2483097,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10800
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034841.092, "dur": 0.727,
    "args": {
      "External id": 2483098,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10801
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034842.249, "dur": 3.042,
    "args": {
      "External id": 2483099,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10802
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034844.631, "dur": 0.595,
    "args": {
      "External id": 2483100,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10803
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034845.567, "dur": 2.236,
    "args": {
      "External id": 2483101,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10804
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034847.148, "dur": 0.593,
    "args": {
      "External id": 2483102,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10805
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034848.166, "dur": 1.503,
    "args": {
      "External id": 2483103,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10806
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034848.805, "dur": 0.791,
    "args": {
      "External id": 2483104,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10807
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034850.028, "dur": 2.802,
    "args": {
      "External id": 2483105,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10808
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034852.248, "dur": 0.518,
    "args": {
      "External id": 2483106,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10809
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034853.178, "dur": 1.732,
    "args": {
      "External id": 2483107,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10810
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034854.210, "dur": 0.634,
    "args": {
      "External id": 2483108,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10811
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034855.267, "dur": 3.031,
    "args": {
      "External id": 2483109,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10812
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034856.363, "dur": 1.856,
    "args": {
      "External id": 2483110,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10813
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034860.008, "dur": 2.131,
    "args": {
      "External id": 2483111,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10814
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034861.395, "dur": 0.675,
    "args": {
      "External id": 2483112,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10815
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034862.399, "dur": 1.462,
    "args": {
      "External id": 2483113,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10816
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034863.111, "dur": 0.672,
    "args": {
      "External id": 2483114,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10817
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034864.187, "dur": 3.528,
    "args": {
      "External id": 2483115,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10818
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034867.214, "dur": 0.440,
    "args": {
      "External id": 2483116,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10819
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034868.105, "dur": 1.714,
    "args": {
      "External id": 2483117,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10820
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034869.078, "dur": 0.670,
    "args": {
      "External id": 2483118,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10821
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034870.190, "dur": 2.163,
    "args": {
      "External id": 2483119,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10822
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034871.599, "dur": 0.680,
    "args": {
      "External id": 2483120,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10823
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034872.821, "dur": 2.609,
    "args": {
      "External id": 2483121,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10824
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034874.891, "dur": 0.472,
    "args": {
      "External id": 2483122,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10825
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034875.662, "dur": 1.695,
    "args": {
      "External id": 2483123,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10826
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034876.780, "dur": 0.515,
    "args": {
      "External id": 2483124,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10827
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034877.886, "dur": 2.715,
    "args": {
      "External id": 2483125,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10828
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034878.688, "dur": 1.840,
    "args": {
      "External id": 2483126,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10829
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034882.483, "dur": 1.507,
    "args": {
      "External id": 2483127,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10830
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034883.399, "dur": 0.531,
    "args": {
      "External id": 2483128,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10831
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034884.542, "dur": 1.528,
    "args": {
      "External id": 2483129,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10832
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034885.395, "dur": 0.603,
    "args": {
      "External id": 2483130,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10833
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034886.446, "dur": 2.918,
    "args": {
      "External id": 2483131,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10834
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034888.865, "dur": 0.431,
    "args": {
      "External id": 2483132,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10835
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034889.780, "dur": 1.369,
    "args": {
      "External id": 2483133,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10836
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034890.489, "dur": 0.597,
    "args": {
      "External id": 2483134,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10837
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034891.386, "dur": 1.952,
    "args": {
      "External id": 2483135,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10838
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034892.508, "dur": 0.669,
    "args": {
      "External id": 2483136,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10839
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034893.599, "dur": 3.060,
    "args": {
      "External id": 2483137,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10840
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034896.086, "dur": 0.505,
    "args": {
      "External id": 2483138,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10841
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034896.947, "dur": 1.712,
    "args": {
      "External id": 2483139,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10842
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034897.890, "dur": 0.707,
    "args": {
      "External id": 2483140,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10843
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034898.916, "dur": 2.943,
    "args": {
      "External id": 2483141,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10844
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034899.677, "dur": 2.108,
    "args": {
      "External id": 2483142,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10845
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034904.083, "dur": 1.421,
    "args": {
      "External id": 2483143,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10846
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034904.981, "dur": 0.459,
    "args": {
      "External id": 2483144,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10847
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034905.941, "dur": 1.794,
    "args": {
      "External id": 2483145,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10848
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034906.854, "dur": 0.680,
    "args": {
      "External id": 2483146,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10849
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034908.050, "dur": 3.126,
    "args": {
      "External id": 2483147,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10850
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034910.636, "dur": 0.478,
    "args": {
      "External id": 2483148,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10851
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034911.541, "dur": 1.638,
    "args": {
      "External id": 2483149,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10852
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034912.476, "dur": 0.642,
    "args": {
      "External id": 2483150,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10853
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034913.412, "dur": 2.059,
    "args": {
      "External id": 2483151,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10854
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034914.762, "dur": 0.639,
    "args": {
      "External id": 2483152,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10855
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034915.730, "dur": 2.643,
    "args": {
      "External id": 2483153,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10856
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034917.389, "dur": 0.806,
    "args": {
      "External id": 2483154,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10857
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034918.603, "dur": 2.236,
    "args": {
      "External id": 2483155,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10858
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034919.618, "dur": 1.012,
    "args": {
      "External id": 2483156,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10859
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034921.094, "dur": 2.884,
    "args": {
      "External id": 2483157,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10860
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034921.744, "dur": 2.014,
    "args": {
      "External id": 2483158,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10861
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034925.708, "dur": 1.595,
    "args": {
      "External id": 2483159,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10862
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034926.743, "dur": 0.402,
    "args": {
      "External id": 2483160,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10863
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034927.699, "dur": 1.498,
    "args": {
      "External id": 2483161,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10864
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034928.364, "dur": 0.602,
    "args": {
      "External id": 2483162,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10865
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034929.527, "dur": 3.478,
    "args": {
      "External id": 2483163,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10866
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034932.541, "dur": 0.394,
    "args": {
      "External id": 2483164,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10867
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034933.388, "dur": 1.795,
    "args": {
      "External id": 2483165,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10868
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034934.343, "dur": 0.622,
    "args": {
      "External id": 2483166,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10869
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034935.531, "dur": 1.540,
    "args": {
      "External id": 2483167,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10870
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034936.448, "dur": 0.553,
    "args": {
      "External id": 2483168,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10871
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034937.470, "dur": 3.096,
    "args": {
      "External id": 2483169,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10872
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034939.856, "dur": 0.646,
    "args": {
      "External id": 2483170,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10873
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034940.804, "dur": 1.493,
    "args": {
      "External id": 2483171,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10874
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034941.688, "dur": 0.544,
    "args": {
      "External id": 2483172,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10875
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034942.563, "dur": 2.367,
    "args": {
      "External id": 2483173,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10876
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034943.124, "dur": 1.730,
    "args": {
      "External id": 2483174,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10877
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034946.878, "dur": 1.535,
    "args": {
      "External id": 2483175,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10878
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034947.708, "dur": 0.642,
    "args": {
      "External id": 2483176,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10879
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034948.668, "dur": 1.449,
    "args": {
      "External id": 2483177,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10880
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034949.256, "dur": 0.793,
    "args": {
      "External id": 2483178,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10881
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034950.350, "dur": 2.720,
    "args": {
      "External id": 2483179,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10882
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034952.479, "dur": 0.526,
    "args": {
      "External id": 2483180,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10883
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034953.464, "dur": 1.783,
    "args": {
      "External id": 2483181,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10884
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034954.530, "dur": 0.655,
    "args": {
      "External id": 2483182,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10885
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034955.646, "dur": 1.930,
    "args": {
      "External id": 2483183,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10886
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034956.668, "dur": 0.839,
    "args": {
      "External id": 2483184,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10887
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034957.994, "dur": 3.404,
    "args": {
      "External id": 2483185,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10888
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034960.623, "dur": 0.712,
    "args": {
      "External id": 2483186,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10889
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034961.662, "dur": 1.821,
    "args": {
      "External id": 2483187,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10890
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034962.759, "dur": 0.663,
    "args": {
      "External id": 2483188,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10891
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034963.740, "dur": 2.891,
    "args": {
      "External id": 2483189,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10892
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034964.369, "dur": 2.074,
    "args": {
      "External id": 2483190,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10893
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034968.502, "dur": 1.760,
    "args": {
      "External id": 2483191,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10894
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034969.395, "dur": 0.639,
    "args": {
      "External id": 2483192,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10895
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034970.520, "dur": 1.269,
    "args": {
      "External id": 2483193,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10896
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034971.077, "dur": 0.641,
    "args": {
      "External id": 2483194,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10897
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034972.022, "dur": 3.233,
    "args": {
      "External id": 2483195,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10898
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034974.710, "dur": 0.479,
    "args": {
      "External id": 2483196,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10899
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034975.621, "dur": 1.600,
    "args": {
      "External id": 2483197,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10900
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034976.451, "dur": 0.705,
    "args": {
      "External id": 2483198,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10901
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034977.562, "dur": 1.643,
    "args": {
      "External id": 2483199,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10902
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658034978.518, "dur": 0.617,
    "args": {
      "External id": 2483200,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10903
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035030.005, "dur": 5.301,
    "args": {
      "External id": 2489345,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10904
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035034.197, "dur": 0.822,
    "args": {
      "External id": 2489346,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10905
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035035.688, "dur": 1.393,
    "args": {
      "External id": 2489347,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10906
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035036.469, "dur": 0.544,
    "args": {
      "External id": 2489348,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10907
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035037.386, "dur": 3.147,
    "args": {
      "External id": 2489349,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10908
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035038.077, "dur": 2.155,
    "args": {
      "External id": 2489350,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10909
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035042.575, "dur": 1.667,
    "args": {
      "External id": 2489351,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10910
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035043.453, "dur": 0.684,
    "args": {
      "External id": 2489352,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10911
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035044.522, "dur": 1.543,
    "args": {
      "External id": 2489353,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10912
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035045.225, "dur": 0.762,
    "args": {
      "External id": 2489354,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10913
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035046.323, "dur": 3.026,
    "args": {
      "External id": 2489355,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10914
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035048.825, "dur": 0.440,
    "args": {
      "External id": 2489356,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10915
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035049.680, "dur": 1.335,
    "args": {
      "External id": 2489357,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10916
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035050.263, "dur": 0.682,
    "args": {
      "External id": 2489358,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10917
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035051.285, "dur": 1.716,
    "args": {
      "External id": 2489359,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10918
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035052.149, "dur": 0.761,
    "args": {
      "External id": 2489360,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10919
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035053.342, "dur": 2.771,
    "args": {
      "External id": 2489361,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10920
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035055.523, "dur": 0.525,
    "args": {
      "External id": 2489362,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10921
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035056.375, "dur": 1.589,
    "args": {
      "External id": 2489363,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10922
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035057.475, "dur": 0.412,
    "args": {
      "External id": 2489364,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10923
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035058.238, "dur": 2.580,
    "args": {
      "External id": 2489365,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10924
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035058.925, "dur": 1.810,
    "args": {
      "External id": 2489366,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10925
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035062.915, "dur": 1.689,
    "args": {
      "External id": 2489367,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10926
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035063.831, "dur": 0.709,
    "args": {
      "External id": 2489368,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10927
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035064.876, "dur": 1.408,
    "args": {
      "External id": 2489369,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10928
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035065.476, "dur": 0.732,
    "args": {
      "External id": 2489370,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10929
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035066.664, "dur": 3.124,
    "args": {
      "External id": 2489371,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10930
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035069.129, "dur": 0.594,
    "args": {
      "External id": 2489372,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10931
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035070.097, "dur": 1.569,
    "args": {
      "External id": 2489373,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10932
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035070.890, "dur": 0.714,
    "args": {
      "External id": 2489374,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10933
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035071.935, "dur": 1.736,
    "args": {
      "External id": 2489375,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10934
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035073.000, "dur": 0.578,
    "args": {
      "External id": 2489376,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10935
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035073.942, "dur": 2.862,
    "args": {
      "External id": 2489377,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10936
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035076.081, "dur": 0.663,
    "args": {
      "External id": 2489378,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10937
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035077.093, "dur": 1.501,
    "args": {
      "External id": 2489379,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10938
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035077.969, "dur": 0.563,
    "args": {
      "External id": 2489380,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10939
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035078.863, "dur": 2.771,
    "args": {
      "External id": 2489381,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10940
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035079.525, "dur": 2.027,
    "args": {
      "External id": 2489382,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10941
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035083.797, "dur": 1.373,
    "args": {
      "External id": 2489383,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10942
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035084.603, "dur": 0.501,
    "args": {
      "External id": 2489384,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10943
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035085.456, "dur": 1.588,
    "args": {
      "External id": 2489385,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10944
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035086.156, "dur": 0.812,
    "args": {
      "External id": 2489386,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10945
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035087.294, "dur": 3.148,
    "args": {
      "External id": 2489387,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10946
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035089.986, "dur": 0.392,
    "args": {
      "External id": 2489388,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10947
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035090.755, "dur": 1.340,
    "args": {
      "External id": 2489389,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10948
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035091.417, "dur": 0.587,
    "args": {
      "External id": 2489390,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10949
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035092.344, "dur": 1.799,
    "args": {
      "External id": 2489391,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10950
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035093.357, "dur": 0.686,
    "args": {
      "External id": 2489392,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10951
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035094.443, "dur": 2.785,
    "args": {
      "External id": 2489393,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10952
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035096.628, "dur": 0.536,
    "args": {
      "External id": 2489394,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10953
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035097.523, "dur": 1.552,
    "args": {
      "External id": 2489395,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10954
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035098.453, "dur": 0.562,
    "args": {
      "External id": 2489396,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10955
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035099.346, "dur": 2.460,
    "args": {
      "External id": 2489397,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10956
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035100.060, "dur": 1.666,
    "args": {
      "External id": 2489398,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10957
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035103.742, "dur": 1.687,
    "args": {
      "External id": 2489399,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10958
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035104.637, "dur": 0.729,
    "args": {
      "External id": 2489400,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10959
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035105.846, "dur": 1.536,
    "args": {
      "External id": 2489401,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10960
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035106.507, "dur": 0.801,
    "args": {
      "External id": 2489402,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10961
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035107.655, "dur": 3.199,
    "args": {
      "External id": 2489403,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10962
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035110.130, "dur": 0.659,
    "args": {
      "External id": 2489404,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10963
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035111.162, "dur": 1.405,
    "args": {
      "External id": 2489405,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10964
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035111.922, "dur": 0.546,
    "args": {
      "External id": 2489406,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10965
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035112.813, "dur": 1.894,
    "args": {
      "External id": 2489407,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10966
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035114.004, "dur": 0.625,
    "args": {
      "External id": 2489408,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10967
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035114.976, "dur": 2.826,
    "args": {
      "External id": 2489409,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10968
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035117.140, "dur": 0.600,
    "args": {
      "External id": 2489410,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10969
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035118.196, "dur": 1.692,
    "args": {
      "External id": 2489411,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10970
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035119.133, "dur": 0.693,
    "args": {
      "External id": 2489412,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10971
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035120.264, "dur": 2.695,
    "args": {
      "External id": 2489413,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10972
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035121.137, "dur": 1.724,
    "args": {
      "External id": 2489414,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10973
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035124.781, "dur": 1.618,
    "args": {
      "External id": 2489415,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10974
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035125.864, "dur": 0.445,
    "args": {
      "External id": 2489416,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10975
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035126.668, "dur": 1.400,
    "args": {
      "External id": 2489417,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10976
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035127.242, "dur": 0.746,
    "args": {
      "External id": 2489418,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10977
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035128.332, "dur": 3.389,
    "args": {
      "External id": 2489419,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10978
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035131.150, "dur": 0.507,
    "args": {
      "External id": 2489420,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10979
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035132.013, "dur": 1.414,
    "args": {
      "External id": 2489421,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10980
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035132.814, "dur": 0.550,
    "args": {
      "External id": 2489422,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10981
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035133.748, "dur": 1.622,
    "args": {
      "External id": 2489423,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10982
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035134.565, "dur": 0.712,
    "args": {
      "External id": 2489424,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10983
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035135.638, "dur": 2.496,
    "args": {
      "External id": 2489425,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10984
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035137.567, "dur": 0.506,
    "args": {
      "External id": 2489426,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10985
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035138.401, "dur": 1.682,
    "args": {
      "External id": 2489427,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10986
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035139.338, "dur": 0.656,
    "args": {
      "External id": 2489428,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10987
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035140.434, "dur": 2.163,
    "args": {
      "External id": 2489429,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10988
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035141.037, "dur": 1.477,
    "args": {
      "External id": 2489430,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10989
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035144.228, "dur": 1.450,
    "args": {
      "External id": 2489431,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10990
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035145.007, "dur": 0.603,
    "args": {
      "External id": 2489432,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10991
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035145.948, "dur": 1.460,
    "args": {
      "External id": 2489433,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10992
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035146.586, "dur": 0.743,
    "args": {
      "External id": 2489434,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10993
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035147.653, "dur": 3.375,
    "args": {
      "External id": 2489435,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10994
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035150.400, "dur": 0.565,
    "args": {
      "External id": 2489436,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10995
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035151.321, "dur": 1.360,
    "args": {
      "External id": 2489437,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10996
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035152.016, "dur": 0.598,
    "args": {
      "External id": 2489438,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10997
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035153.075, "dur": 1.493,
    "args": {
      "External id": 2489439,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 10998
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035153.915, "dur": 0.560,
    "args": {
      "External id": 2489440,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 10999
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035154.960, "dur": 2.422,
    "args": {
      "External id": 2489441,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11000
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035156.810, "dur": 0.510,
    "args": {
      "External id": 2489442,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11001
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035157.626, "dur": 1.597,
    "args": {
      "External id": 2489443,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11002
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035158.601, "dur": 0.528,
    "args": {
      "External id": 2489444,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11003
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035159.493, "dur": 2.776,
    "args": {
      "External id": 2489445,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11004
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035160.236, "dur": 1.952,
    "args": {
      "External id": 2489446,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11005
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035163.988, "dur": 1.545,
    "args": {
      "External id": 2489447,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11006
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035164.897, "dur": 0.575,
    "args": {
      "External id": 2489448,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11007
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035165.898, "dur": 1.277,
    "args": {
      "External id": 2489449,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11008
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035166.488, "dur": 0.609,
    "args": {
      "External id": 2489450,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11009
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035167.419, "dur": 2.980,
    "args": {
      "External id": 2489451,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11010
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035169.598, "dur": 0.735,
    "args": {
      "External id": 2489452,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11011
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035170.704, "dur": 1.590,
    "args": {
      "External id": 2489453,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11012
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035171.436, "dur": 0.796,
    "args": {
      "External id": 2489454,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11013
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035172.539, "dur": 1.580,
    "args": {
      "External id": 2489455,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11014
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035173.448, "dur": 0.579,
    "args": {
      "External id": 2489456,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11015
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035174.392, "dur": 2.806,
    "args": {
      "External id": 2489457,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11016
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035176.503, "dur": 0.607,
    "args": {
      "External id": 2489458,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11017
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035177.590, "dur": 1.655,
    "args": {
      "External id": 2489459,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11018
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035178.559, "dur": 0.624,
    "args": {
      "External id": 2489460,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11019
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035179.517, "dur": 2.519,
    "args": {
      "External id": 2489461,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11020
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035180.165, "dur": 1.790,
    "args": {
      "External id": 2489462,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11021
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035183.904, "dur": 1.866,
    "args": {
      "External id": 2489463,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11022
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035185.014, "dur": 0.694,
    "args": {
      "External id": 2489464,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11023
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035186.144, "dur": 1.405,
    "args": {
      "External id": 2489465,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11024
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035186.775, "dur": 0.697,
    "args": {
      "External id": 2489466,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11025
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035187.795, "dur": 2.973,
    "args": {
      "External id": 2489467,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11026
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035190.120, "dur": 0.588,
    "args": {
      "External id": 2489468,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11027
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035191.059, "dur": 1.206,
    "args": {
      "External id": 2489469,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11028
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035191.690, "dur": 0.509,
    "args": {
      "External id": 2489470,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11029
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035192.510, "dur": 1.696,
    "args": {
      "External id": 2489471,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11030
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035193.481, "dur": 0.649,
    "args": {
      "External id": 2489472,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11031
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035194.479, "dur": 2.569,
    "args": {
      "External id": 2489473,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11032
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035196.339, "dur": 0.647,
    "args": {
      "External id": 2489474,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11033
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035197.292, "dur": 2.780,
    "args": {
      "External id": 2489475,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11034
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035199.450, "dur": 0.562,
    "args": {
      "External id": 2489476,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11035
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035200.494, "dur": 2.672,
    "args": {
      "External id": 2489477,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11036
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035201.218, "dur": 1.870,
    "args": {
      "External id": 2489478,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11037
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035204.807, "dur": 1.404,
    "args": {
      "External id": 2489479,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11038
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035205.611, "dur": 0.534,
    "args": {
      "External id": 2489480,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11039
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035206.724, "dur": 1.250,
    "args": {
      "External id": 2489481,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11040
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035207.320, "dur": 0.579,
    "args": {
      "External id": 2489482,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11041
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035208.222, "dur": 3.218,
    "args": {
      "External id": 2489483,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11042
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035210.828, "dur": 0.546,
    "args": {
      "External id": 2489484,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11043
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035211.732, "dur": 1.154,
    "args": {
      "External id": 2489485,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11044
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035212.305, "dur": 0.519,
    "args": {
      "External id": 2489486,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11045
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035213.153, "dur": 1.575,
    "args": {
      "External id": 2489487,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11046
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035213.908, "dur": 0.745,
    "args": {
      "External id": 2489488,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11047
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035215.116, "dur": 2.693,
    "args": {
      "External id": 2489489,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11048
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035217.028, "dur": 0.693,
    "args": {
      "External id": 2489490,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11049
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035218.126, "dur": 1.665,
    "args": {
      "External id": 2489491,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11050
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035219.268, "dur": 0.436,
    "args": {
      "External id": 2489492,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11051
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035220.061, "dur": 2.148,
    "args": {
      "External id": 2489493,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11052
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035220.629, "dur": 1.502,
    "args": {
      "External id": 2489494,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11053
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035223.990, "dur": 1.448,
    "args": {
      "External id": 2489495,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11054
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035224.841, "dur": 0.534,
    "args": {
      "External id": 2489496,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11055
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035225.935, "dur": 1.314,
    "args": {
      "External id": 2489497,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11056
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035226.590, "dur": 0.583,
    "args": {
      "External id": 2489498,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11057
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035227.519, "dur": 3.256,
    "args": {
      "External id": 2489499,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11058
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035230.154, "dur": 0.560,
    "args": {
      "External id": 2489500,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11059
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035231.065, "dur": 1.523,
    "args": {
      "External id": 2489501,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11060
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035231.811, "dur": 0.712,
    "args": {
      "External id": 2489502,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11061
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035232.832, "dur": 1.634,
    "args": {
      "External id": 2489503,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11062
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035233.781, "dur": 0.595,
    "args": {
      "External id": 2489504,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11063
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035234.736, "dur": 2.510,
    "args": {
      "External id": 2489505,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11064
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035236.669, "dur": 0.513,
    "args": {
      "External id": 2489506,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11065
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035237.510, "dur": 1.995,
    "args": {
      "External id": 2489507,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11066
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035238.750, "dur": 0.691,
    "args": {
      "External id": 2489508,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11067
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035239.774, "dur": 2.415,
    "args": {
      "External id": 2489509,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11068
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035240.361, "dur": 1.747,
    "args": {
      "External id": 2489510,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11069
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035244.983, "dur": 1.556,
    "args": {
      "External id": 2489511,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11070
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035245.926, "dur": 0.551,
    "args": {
      "External id": 2489512,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11071
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035246.908, "dur": 1.580,
    "args": {
      "External id": 2489513,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11072
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035247.609, "dur": 0.803,
    "args": {
      "External id": 2489514,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11073
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035248.754, "dur": 3.042,
    "args": {
      "External id": 2489515,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11074
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035251.181, "dur": 0.551,
    "args": {
      "External id": 2489516,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11075
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035252.087, "dur": 1.202,
    "args": {
      "External id": 2489517,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11076
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035252.655, "dur": 0.570,
    "args": {
      "External id": 2489518,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11077
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035253.666, "dur": 1.463,
    "args": {
      "External id": 2489519,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11078
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035254.622, "dur": 0.437,
    "args": {
      "External id": 2489520,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11079
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035255.545, "dur": 2.388,
    "args": {
      "External id": 2489521,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11080
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035257.289, "dur": 0.579,
    "args": {
      "External id": 2489522,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11081
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035258.197, "dur": 1.730,
    "args": {
      "External id": 2489523,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11082
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035259.265, "dur": 0.578,
    "args": {
      "External id": 2489524,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11083
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035260.241, "dur": 2.964,
    "args": {
      "External id": 2489525,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11084
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035261.029, "dur": 1.943,
    "args": {
      "External id": 2489526,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11085
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035264.848, "dur": 1.571,
    "args": {
      "External id": 2489527,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11086
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035265.768, "dur": 0.560,
    "args": {
      "External id": 2489528,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11087
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035266.773, "dur": 1.442,
    "args": {
      "External id": 2489529,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11088
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035267.401, "dur": 0.738,
    "args": {
      "External id": 2489530,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11089
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035268.514, "dur": 3.507,
    "args": {
      "External id": 2489531,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11090
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035271.316, "dur": 0.643,
    "args": {
      "External id": 2489532,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11091
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035272.314, "dur": 1.475,
    "args": {
      "External id": 2489533,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11092
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035273.147, "dur": 0.580,
    "args": {
      "External id": 2489534,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11093
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035274.033, "dur": 1.605,
    "args": {
      "External id": 2489535,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11094
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035274.951, "dur": 0.611,
    "args": {
      "External id": 2489536,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11095
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035275.934, "dur": 2.826,
    "args": {
      "External id": 2489537,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11096
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035277.987, "dur": 0.713,
    "args": {
      "External id": 2489538,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11097
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035279.167, "dur": 1.482,
    "args": {
      "External id": 2489539,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11098
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035280.091, "dur": 0.474,
    "args": {
      "External id": 2489540,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11099
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035280.919, "dur": 2.308,
    "args": {
      "External id": 2489541,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11100
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035281.512, "dur": 1.635,
    "args": {
      "External id": 2489542,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11101
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035285.715, "dur": 1.512,
    "args": {
      "External id": 2489543,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11102
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035286.631, "dur": 0.533,
    "args": {
      "External id": 2489544,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11103
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035287.496, "dur": 1.480,
    "args": {
      "External id": 2489545,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11104
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035288.071, "dur": 0.831,
    "args": {
      "External id": 2489546,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11105
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035289.481, "dur": 3.189,
    "args": {
      "External id": 2489547,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11106
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035292.092, "dur": 0.514,
    "args": {
      "External id": 2489548,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11107
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035292.973, "dur": 1.488,
    "args": {
      "External id": 2489549,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11108
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035293.831, "dur": 0.566,
    "args": {
      "External id": 2489550,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11109
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035294.729, "dur": 1.534,
    "args": {
      "External id": 2489551,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11110
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035295.556, "dur": 0.629,
    "args": {
      "External id": 2489552,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11111
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035296.643, "dur": 2.433,
    "args": {
      "External id": 2489553,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11112
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035298.546, "dur": 0.441,
    "args": {
      "External id": 2489554,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11113
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035299.320, "dur": 1.453,
    "args": {
      "External id": 2489555,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11114
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035300.107, "dur": 0.582,
    "args": {
      "External id": 2489556,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11115
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035301.171, "dur": 2.460,
    "args": {
      "External id": 2489557,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11116
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035301.959, "dur": 1.591,
    "args": {
      "External id": 2489558,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11117
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035305.160, "dur": 1.612,
    "args": {
      "External id": 2489559,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11118
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035306.153, "dur": 0.558,
    "args": {
      "External id": 2489560,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11119
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035307.316, "dur": 1.651,
    "args": {
      "External id": 2489561,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11120
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035308.119, "dur": 0.773,
    "args": {
      "External id": 2489562,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11121
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035309.231, "dur": 3.226,
    "args": {
      "External id": 2489563,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11122
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035311.843, "dur": 0.553,
    "args": {
      "External id": 2489564,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11123
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035312.748, "dur": 1.285,
    "args": {
      "External id": 2489565,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11124
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035313.323, "dur": 0.645,
    "args": {
      "External id": 2489566,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11125
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035314.277, "dur": 1.591,
    "args": {
      "External id": 2489567,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11126
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035315.139, "dur": 0.652,
    "args": {
      "External id": 2489568,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11127
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035316.136, "dur": 2.560,
    "args": {
      "External id": 2489569,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11128
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035318.104, "dur": 0.531,
    "args": {
      "External id": 2489570,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11129
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035318.978, "dur": 1.508,
    "args": {
      "External id": 2489571,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11130
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035319.871, "dur": 0.554,
    "args": {
      "External id": 2489572,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11131
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035320.756, "dur": 2.422,
    "args": {
      "External id": 2489573,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11132
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035321.496, "dur": 1.601,
    "args": {
      "External id": 2489574,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11133
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035324.995, "dur": 1.604,
    "args": {
      "External id": 2489575,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11134
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035325.872, "dur": 0.664,
    "args": {
      "External id": 2489576,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11135
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035326.869, "dur": 1.450,
    "args": {
      "External id": 2489577,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11136
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035327.452, "dur": 0.791,
    "args": {
      "External id": 2489578,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11137
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035328.586, "dur": 3.277,
    "args": {
      "External id": 2489579,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11138
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035331.249, "dur": 0.547,
    "args": {
      "External id": 2489580,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11139
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035332.175, "dur": 1.266,
    "args": {
      "External id": 2489581,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11140
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035332.862, "dur": 0.511,
    "args": {
      "External id": 2489582,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11141
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035333.764, "dur": 1.703,
    "args": {
      "External id": 2489583,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11142
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035334.671, "dur": 0.719,
    "args": {
      "External id": 2489584,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11143
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035335.756, "dur": 2.407,
    "args": {
      "External id": 2489585,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11144
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035337.560, "dur": 0.536,
    "args": {
      "External id": 2489586,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11145
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035338.423, "dur": 1.534,
    "args": {
      "External id": 2489587,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11146
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035339.284, "dur": 0.613,
    "args": {
      "External id": 2489588,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11147
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035340.230, "dur": 2.268,
    "args": {
      "External id": 2489589,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11148
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035340.874, "dur": 1.544,
    "args": {
      "External id": 2489590,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11149
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035344.555, "dur": 1.763,
    "args": {
      "External id": 2489591,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11150
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035345.677, "dur": 0.576,
    "args": {
      "External id": 2489592,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11151
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035346.604, "dur": 1.380,
    "args": {
      "External id": 2489593,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11152
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035347.240, "dur": 0.666,
    "args": {
      "External id": 2489594,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11153
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035348.251, "dur": 2.725,
    "args": {
      "External id": 2489595,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11154
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035350.219, "dur": 0.692,
    "args": {
      "External id": 2489596,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11155
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035351.269, "dur": 1.280,
    "args": {
      "External id": 2489597,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11156
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035351.840, "dur": 0.643,
    "args": {
      "External id": 2489598,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11157
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035352.910, "dur": 1.728,
    "args": {
      "External id": 2489599,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11158
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035353.823, "dur": 0.737,
    "args": {
      "External id": 2489600,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11159
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035355.027, "dur": 2.496,
    "args": {
      "External id": 2489601,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11160
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035356.945, "dur": 0.510,
    "args": {
      "External id": 2489602,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11161
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035357.843, "dur": 1.996,
    "args": {
      "External id": 2489603,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11162
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035359.073, "dur": 0.700,
    "args": {
      "External id": 2489604,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11163
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035360.142, "dur": 2.880,
    "args": {
      "External id": 2489605,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11164
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035360.917, "dur": 1.938,
    "args": {
      "External id": 2489606,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11165
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035364.885, "dur": 1.553,
    "args": {
      "External id": 2489607,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11166
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035365.756, "dur": 0.617,
    "args": {
      "External id": 2489608,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11167
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035366.720, "dur": 1.279,
    "args": {
      "External id": 2489609,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11168
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035367.352, "dur": 0.567,
    "args": {
      "External id": 2489610,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11169
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035368.243, "dur": 2.933,
    "args": {
      "External id": 2489611,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11170
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035370.602, "dur": 0.512,
    "args": {
      "External id": 2489612,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11171
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035371.469, "dur": 1.296,
    "args": {
      "External id": 2489613,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11172
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035372.089, "dur": 0.614,
    "args": {
      "External id": 2489614,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11173
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035373.092, "dur": 1.851,
    "args": {
      "External id": 2489615,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11174
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035374.193, "dur": 0.674,
    "args": {
      "External id": 2489616,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11175
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035375.296, "dur": 2.500,
    "args": {
      "External id": 2489617,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11176
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035377.133, "dur": 0.600,
    "args": {
      "External id": 2489618,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11177
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035378.182, "dur": 1.612,
    "args": {
      "External id": 2489619,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11178
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035379.049, "dur": 0.656,
    "args": {
      "External id": 2489620,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11179
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035380.150, "dur": 2.806,
    "args": {
      "External id": 2489621,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11180
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035380.867, "dur": 1.868,
    "args": {
      "External id": 2489622,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11181
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035384.729, "dur": 1.426,
    "args": {
      "External id": 2489623,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11182
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035385.647, "dur": 0.445,
    "args": {
      "External id": 2489624,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11183
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035386.448, "dur": 1.260,
    "args": {
      "External id": 2489625,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11184
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035387.013, "dur": 0.620,
    "args": {
      "External id": 2489626,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11185
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035388.067, "dur": 3.153,
    "args": {
      "External id": 2489627,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11186
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035390.602, "dur": 0.522,
    "args": {
      "External id": 2489628,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11187
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035391.535, "dur": 1.449,
    "args": {
      "External id": 2489629,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11188
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035392.293, "dur": 0.631,
    "args": {
      "External id": 2489630,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11189
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035393.228, "dur": 1.810,
    "args": {
      "External id": 2489631,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11190
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035394.268, "dur": 0.696,
    "args": {
      "External id": 2489632,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11191
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035395.329, "dur": 2.558,
    "args": {
      "External id": 2489633,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11192
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035397.285, "dur": 0.525,
    "args": {
      "External id": 2489634,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11193
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035398.132, "dur": 1.748,
    "args": {
      "External id": 2489635,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11194
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035399.137, "dur": 0.522,
    "args": {
      "External id": 2489636,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11195
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035400.148, "dur": 2.980,
    "args": {
      "External id": 2489637,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11196
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035400.811, "dur": 2.235,
    "args": {
      "External id": 2489638,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11197
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035404.852, "dur": 1.493,
    "args": {
      "External id": 2489639,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11198
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035405.631, "dur": 0.652,
    "args": {
      "External id": 2489640,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11199
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035406.612, "dur": 1.594,
    "args": {
      "External id": 2489641,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11200
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035407.294, "dur": 0.831,
    "args": {
      "External id": 2489642,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11201
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035408.448, "dur": 3.748,
    "args": {
      "External id": 2489643,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11202
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035411.433, "dur": 0.699,
    "args": {
      "External id": 2489644,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11203
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035412.486, "dur": 1.248,
    "args": {
      "External id": 2489645,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11204
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035413.128, "dur": 0.541,
    "args": {
      "External id": 2489646,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11205
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035413.991, "dur": 1.596,
    "args": {
      "External id": 2489647,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11206
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035414.824, "dur": 0.688,
    "args": {
      "External id": 2489648,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11207
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035415.876, "dur": 3.135,
    "args": {
      "External id": 2489649,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11208
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035418.085, "dur": 0.860,
    "args": {
      "External id": 2489650,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11209
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035419.252, "dur": 1.628,
    "args": {
      "External id": 2489651,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11210
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035420.178, "dur": 0.637,
    "args": {
      "External id": 2489652,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11211
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035421.264, "dur": 2.645,
    "args": {
      "External id": 2489653,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11212
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035422.082, "dur": 1.745,
    "args": {
      "External id": 2489654,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11213
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035426.015, "dur": 1.715,
    "args": {
      "External id": 2489655,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11214
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035427.014, "dur": 0.652,
    "args": {
      "External id": 2489656,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11215
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035428.000, "dur": 1.729,
    "args": {
      "External id": 2489657,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11216
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035428.780, "dur": 0.867,
    "args": {
      "External id": 2489658,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11217
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035430.109, "dur": 3.369,
    "args": {
      "External id": 2489659,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11218
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035432.644, "dur": 0.767,
    "args": {
      "External id": 2489660,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11219
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035433.768, "dur": 1.438,
    "args": {
      "External id": 2489661,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11220
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035434.598, "dur": 0.542,
    "args": {
      "External id": 2489662,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11221
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035435.471, "dur": 2.053,
    "args": {
      "External id": 2489663,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11222
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035436.643, "dur": 0.804,
    "args": {
      "External id": 2489664,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11223
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035437.792, "dur": 2.387,
    "args": {
      "External id": 2489665,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11224
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035439.553, "dur": 0.562,
    "args": {
      "External id": 2489666,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11225
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035440.420, "dur": 1.701,
    "args": {
      "External id": 2489667,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11226
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035441.453, "dur": 0.571,
    "args": {
      "External id": 2489668,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11227
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035442.393, "dur": 2.726,
    "args": {
      "External id": 2489669,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11228
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035443.118, "dur": 1.919,
    "args": {
      "External id": 2489670,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11229
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035446.495, "dur": 1.488,
    "args": {
      "External id": 2489671,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11230
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035447.358, "dur": 0.556,
    "args": {
      "External id": 2489672,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11231
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035448.371, "dur": 1.453,
    "args": {
      "External id": 2489673,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11232
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035448.939, "dur": 0.806,
    "args": {
      "External id": 2489674,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11233
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035450.064, "dur": 2.724,
    "args": {
      "External id": 2489675,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11234
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035452.269, "dur": 0.453,
    "args": {
      "External id": 2489676,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11235
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035453.179, "dur": 1.402,
    "args": {
      "External id": 2489677,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11236
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035453.949, "dur": 0.570,
    "args": {
      "External id": 2489678,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11237
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035454.978, "dur": 1.695,
    "args": {
      "External id": 2489679,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11238
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035455.919, "dur": 0.679,
    "args": {
      "External id": 2489680,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11239
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035457.100, "dur": 2.338,
    "args": {
      "External id": 2489681,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11240
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035458.881, "dur": 0.465,
    "args": {
      "External id": 2489682,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11241
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035459.681, "dur": 1.881,
    "args": {
      "External id": 2489683,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11242
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035460.838, "dur": 0.631,
    "args": {
      "External id": 2489684,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11243
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035461.828, "dur": 2.716,
    "args": {
      "External id": 2489685,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11244
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035462.562, "dur": 1.819,
    "args": {
      "External id": 2489686,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11245
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035466.588, "dur": 1.890,
    "args": {
      "External id": 2489687,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11246
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035467.852, "dur": 0.561,
    "args": {
      "External id": 2489688,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11247
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035468.828, "dur": 1.268,
    "args": {
      "External id": 2489689,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11248
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035469.403, "dur": 0.615,
    "args": {
      "External id": 2489690,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11249
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035470.338, "dur": 3.339,
    "args": {
      "External id": 2489691,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11250
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035472.914, "dur": 0.699,
    "args": {
      "External id": 2489692,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11251
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035473.968, "dur": 1.241,
    "args": {
      "External id": 2489693,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11252
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035474.591, "dur": 0.554,
    "args": {
      "External id": 2489694,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11253
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035475.452, "dur": 1.469,
    "args": {
      "External id": 2489695,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11254
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035476.236, "dur": 0.609,
    "args": {
      "External id": 2489696,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11255
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035477.280, "dur": 2.635,
    "args": {
      "External id": 2489697,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11256
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035479.354, "dur": 0.471,
    "args": {
      "External id": 2489698,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11257
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035480.159, "dur": 1.547,
    "args": {
      "External id": 2489699,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11258
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035481.028, "dur": 0.588,
    "args": {
      "External id": 2489700,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11259
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035482.095, "dur": 2.405,
    "args": {
      "External id": 2489701,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11260
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035482.660, "dur": 1.628,
    "args": {
      "External id": 2489702,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11261
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035486.312, "dur": 1.416,
    "args": {
      "External id": 2489703,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11262
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035487.130, "dur": 0.533,
    "args": {
      "External id": 2489704,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11263
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035488.085, "dur": 1.201,
    "args": {
      "External id": 2489705,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11264
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035488.737, "dur": 0.471,
    "args": {
      "External id": 2489706,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11265
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035489.528, "dur": 3.282,
    "args": {
      "External id": 2489707,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11266
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035492.190, "dur": 0.554,
    "args": {
      "External id": 2489708,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11267
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035493.096, "dur": 1.289,
    "args": {
      "External id": 2489709,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11268
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035493.806, "dur": 0.517,
    "args": {
      "External id": 2489710,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11269
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035494.630, "dur": 1.589,
    "args": {
      "External id": 2489711,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11270
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035495.643, "dur": 0.484,
    "args": {
      "External id": 2489712,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11271
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035496.488, "dur": 2.529,
    "args": {
      "External id": 2489713,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11272
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035498.436, "dur": 0.519,
    "args": {
      "External id": 2489714,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11273
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035499.258, "dur": 1.792,
    "args": {
      "External id": 2489715,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11274
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035500.469, "dur": 0.487,
    "args": {
      "External id": 2489716,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11275
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035501.437, "dur": 2.712,
    "args": {
      "External id": 2489717,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11276
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035502.216, "dur": 1.850,
    "args": {
      "External id": 2489718,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11277
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035505.787, "dur": 1.665,
    "args": {
      "External id": 2489719,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11278
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035506.757, "dur": 0.631,
    "args": {
      "External id": 2489720,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11279
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035507.808, "dur": 1.686,
    "args": {
      "External id": 2489721,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11280
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035508.609, "dur": 0.807,
    "args": {
      "External id": 2489722,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11281
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035509.843, "dur": 3.212,
    "args": {
      "External id": 2489723,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11282
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035512.515, "dur": 0.478,
    "args": {
      "External id": 2489724,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11283
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035513.339, "dur": 1.383,
    "args": {
      "External id": 2489725,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11284
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035513.961, "dur": 0.697,
    "args": {
      "External id": 2489726,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11285
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035515.099, "dur": 1.751,
    "args": {
      "External id": 2489727,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11286
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035516.162, "dur": 0.611,
    "args": {
      "External id": 2489728,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11287
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035517.115, "dur": 2.346,
    "args": {
      "External id": 2489729,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11288
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035518.877, "dur": 0.517,
    "args": {
      "External id": 2489730,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11289
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::unsqueeze", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035519.717, "dur": 2.676,
    "args": {
      "External id": 2489731,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11290
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035521.823, "dur": 0.507,
    "args": {
      "External id": 2489732,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[1]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11291
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::cat", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035553.325, "dur": 131.609,
    "args": {
      "External id": 2489733,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11292
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linalg_vector_norm", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035785.477, "dur": 131.135,
    "args": {
      "External id": 2489734,"Record function id": 0, "Concrete Inputs": ["", "2.", "", "False", ""], "Input type": ["float", "Scalar", "", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[291], [], [], [], []], "Ev Idx": 11293
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::linalg_vector_norm", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035850.075, "dur": 46.162,
    "args": {
      "External id": 2489735,"Record function id": 0, "Concrete Inputs": ["", "2.", "", "False", ""], "Input type": ["float", "Scalar", "", "Scalar", ""], "Input Strides": [[1], [], [], [], []], "Input Dims": [[291], [], [], [], []], "Ev Idx": 11294
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::as_strided", "pid": 1336757, "tid": 1336757,
    "ts": 1514658035863.970, "dur": 1.218,
    "args": {
      "External id": 2489736,"Record function id": 0, "Concrete Inputs": ["", "[1]", "[0]", ""], "Input type": ["float", "ScalarList", "ScalarList", ""], "Input Strides": [[], [], [], []], "Input Dims": [[], [], [], []], "Ev Idx": 11295
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "Redistribute", "pid": 1336757, "tid": 1336757,
    "ts": 1514658036345.791, "dur": 1014.676,
    "args": {
      "External id": 2489737,"Sequence number": 24740656, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", "False"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11296
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::pow", "pid": 1336757, "tid": 1336757,
    "ts": 1514658036406.521, "dur": 103.377,
    "args": {
      "External id": 2489738,"Record function id": 0, "Concrete Inputs": ["", "2."], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11297
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658036412.216, "dur": 1.486,
    "args": {
      "External id": 2489739,"Record function id": 0, "Concrete Inputs": ["", "2."], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11298
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658036464.251, "dur": 0.865,
    "args": {
      "External id": 2489740,"Record function id": 0, "Concrete Inputs": ["", "6", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 11299
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "_c10d_functional::all_reduce", "pid": 1336757, "tid": 1336757,
    "ts": 1514658036553.249, "dur": 491.509,
    "args": {
      "External id": 2489741,"Record function id": 0, "Concrete Inputs": ["", "", ""], "Input type": ["float", "", ""], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11300
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clone", "pid": 1336757, "tid": 1336757,
    "ts": 1514658036559.151, "dur": 48.488,
    "args": {
      "External id": 2489742,"Record function id": 0, "Concrete Inputs": ["", "0"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11301
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty_like", "pid": 1336757, "tid": 1336757,
    "ts": 1514658036562.704, "dur": 9.758,
    "args": {
      "External id": 2489743,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "0"], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 11302
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514658036566.337, "dur": 5.001,
    "args": {
      "External id": 2489744,"Record function id": 0, "Concrete Inputs": ["[]", "6", "0", "", "", "0"], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", "Scalar"], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 11303
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::copy_", "pid": 1336757, "tid": 1336757,
    "ts": 1514658036573.931, "dur": 32.932,
    "args": {
      "External id": 2489745,"Record function id": 0, "Concrete Inputs": ["", "", "False"], "Input type": ["float", "float", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11304
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "c10d::allreduce_", "pid": 1336757, "tid": 1336757,
    "ts": 1514658036617.556, "dur": 423.793,
    "args": {
      "External id": 2489746,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "-1"], "Input type": ["TensorList", "", "", "", "Scalar"], "Input Strides": [[[]], [], [], [], []], "Input Dims": [[[]], [], [], [], []], "Ev Idx": 11305
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514658036649.533, "dur": 383.816,
    "args": {
      "External id": 2489747,"Record function id": 0, "Collective name": "allreduce", "Process Group Description": "default_pg", "dtype": "Float", "Rank": 4, "Input Strides": [[[]], [], [], [], [], [], [], [], [], []], "Concrete Inputs": ["", "", "", "4", "", "[]", "[]", "0", "1", "8"], "Out msg nelems": 1, "Process Group Name": "0", "Input type": ["TensorList", "", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Global rank stride": 1, "Out split size": "[]", "In split size": "[]", "Process Group Ranks": "[0, 1, 2, 3, 4, 5, 6, 7]", "Group size": 8, "Input Dims": [[[]], [], [], [], [], [], [], [], [], []], "Global rank start": 0, "Ev Idx": 11306, "In msg nelems": 1
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "nccl:all_reduce", "pid": 1336757, "tid": 1336757,
    "ts": 1514658036668.290, "dur": 358.669,
    "args": {
      "External id": 2489748,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11307
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::pow", "pid": 1336757, "tid": 1336757,
    "ts": 1514658037115.217, "dur": 207.772,
    "args": {
      "External id": 2489749,"Record function id": 0, "Concrete Inputs": ["", "0.5"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11308
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "_c10d_functional::wait_tensor", "pid": 1336757, "tid": 1336757,
    "ts": 1514658037209.128, "dur": 34.069,
    "args": {
      "External id": 2489750,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11309
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "record_param_comms", "pid": 1336757, "tid": 1336757,
    "ts": 1514658037228.525, "dur": 4.559,
    "args": {
      "External id": 2489751,"Record function id": 0, "Collective name": "wait", "Process Group Description": "default_pg", "dtype": "Byte", "Input Dims": [[], [], [], [], [], [], [], [], []], "Input Strides": [[], [], [], [], [], [], [], [], []], "Input type": ["", "", "Scalar", "", "ScalarList", "ScalarList", "Scalar", "Scalar", "Scalar"], "Concrete Inputs": ["", "", "4", "", "[]", "[]", "-1", "-1", "1"], "Out msg nelems": 0, "Process Group Name": "0", "Process Group Ranks": "[]", "Group size": 1, "Ev Idx": 11310, "In msg nelems": 0, "Rank": 4, "In split size": "[]", "Out split size": "[]"
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::pow", "pid": 1336757, "tid": 1336757,
    "ts": 1514658037273.876, "dur": 43.214,
    "args": {
      "External id": 2489752,"Record function id": 0, "Concrete Inputs": ["", "0.5"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11311
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658037276.881, "dur": 1.122,
    "args": {
      "External id": 2489753,"Record function id": 0, "Concrete Inputs": ["", "0.5"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11312
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658037279.388, "dur": 0.546,
    "args": {
      "External id": 2489754,"Record function id": 0, "Concrete Inputs": ["", "6", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 11313
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "_ToTorchTensor", "pid": 1336757, "tid": 1336757,
    "ts": 1514658037375.849, "dur": 21.941,
    "args": {
      "External id": 2489755,"Sequence number": 24740657, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", ""], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11314
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view_as", "pid": 1336757, "tid": 1336757,
    "ts": 1514658037386.584, "dur": 8.044,
    "args": {
      "External id": 2489756,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11315
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::view", "pid": 1336757, "tid": 1336757,
    "ts": 1514658037388.816, "dur": 5.635,
    "args": {
      "External id": 2489757,"Record function id": 0, "Concrete Inputs": ["", "[]"], "Input type": ["float", "ScalarList"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11316
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::add", "pid": 1336757, "tid": 1336757,
    "ts": 1514658037794.049, "dur": 38.898,
    "args": {
      "External id": 2489758,"Record function id": 0, "Concrete Inputs": ["", "", "1"], "Input type": ["float", "double", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11317
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::reciprocal", "pid": 1336757, "tid": 1336757,
    "ts": 1514658037842.676, "dur": 20.651,
    "args": {
      "External id": 2489759,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11318
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::mul", "pid": 1336757, "tid": 1336757,
    "ts": 1514658037870.244, "dur": 21.735,
    "args": {
      "External id": 2489760,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "double"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11319
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::clamp", "pid": 1336757, "tid": 1336757,
    "ts": 1514658037909.046, "dur": 28.617,
    "args": {
      "External id": 2489761,"Record function id": 0, "Concrete Inputs": ["", "", "1."], "Input type": ["float", "", "Scalar"], "Input Strides": [[], [], []], "Input Dims": [[], [], []], "Ev Idx": 11320
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658037913.343, "dur": 0.584,
    "args": {
      "External id": 2489762,"Record function id": 0, "Concrete Inputs": ["", "6", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], []], "Input Dims": [[], [], [], [], []], "Ev Idx": 11321
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::to", "pid": 1336757, "tid": 1336757,
    "ts": 1514658037958.148, "dur": 0.440,
    "args": {
      "External id": 2489763,"Record function id": 0, "Concrete Inputs": ["", "6", "0", "", "", "False", "False", ""], "Input type": ["float", "Scalar", "Scalar", "", "", "Scalar", "Scalar", ""], "Input Strides": [[], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], []], "Ev Idx": 11322
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_mul_", "pid": 1336757, "tid": 1336757,
    "ts": 1514658038115.584, "dur": 806.884,
    "args": {
      "External id": 2489764,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["TensorList", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11323
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_mul_", "pid": 1336757, "tid": 1336757,
    "ts": 1514658038628.869, "dur": 263.567,
    "args": {
      "External id": 2489765,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["TensorList", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11324
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::isnan", "pid": 1336757, "tid": 1336757,
    "ts": 1514658038969.914, "dur": 66.529,
    "args": {
      "External id": 2489766,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11325
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::ne", "pid": 1336757, "tid": 1336757,
    "ts": 1514658038973.714, "dur": 61.826,
    "args": {
      "External id": 2489767,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11326
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336757, "tid": 1336757,
    "ts": 1514658039041.984, "dur": 166.996,
    "args": {
      "External id": 2489768,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11327
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336757, "tid": 1336757,
    "ts": 1514658039043.729, "dur": 165.051,
    "args": {
      "External id": 2489769,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11328
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336757, "tid": 1336757,
    "ts": 1514658039045.620, "dur": 162.731,
    "args": {
      "External id": 2489770,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11329
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::isinf", "pid": 1336757, "tid": 1336757,
    "ts": 1514658039214.274, "dur": 59.493,
    "args": {
      "External id": 2489771,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11330
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514658039217.315, "dur": 33.290,
    "args": {
      "External id": 2489772,"Record function id": 0, "Concrete Inputs": [""], "Input type": ["float"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11331
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::empty", "pid": 1336757, "tid": 1336757,
    "ts": 1514658039224.809, "dur": 3.267,
    "args": {
      "External id": 2489773,"Record function id": 0, "Concrete Inputs": ["[0]", "6", "0", "", "", ""], "Input type": ["ScalarList", "Scalar", "Scalar", "", "", ""], "Input Strides": [[], [], [], [], [], []], "Input Dims": [[], [], [], [], [], []], "Ev Idx": 11332
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::abs", "pid": 1336757, "tid": 1336757,
    "ts": 1514658039229.882, "dur": 20.455,
    "args": {
      "External id": 2489774,"Record function id": 0, "Concrete Inputs": ["", ""], "Input type": ["float", "float"], "Input Strides": [[], [1]], "Input Dims": [[], [0]], "Ev Idx": 11333
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::resize_", "pid": 1336757, "tid": 1336757,
    "ts": 1514658039235.231, "dur": 2.464,
    "args": {
      "External id": 2489775,"Record function id": 0, "Concrete Inputs": ["", "[]", ""], "Input type": ["float", "ScalarList", ""], "Input Strides": [[1], [], []], "Input Dims": [[0], [], []], "Ev Idx": 11334
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::eq", "pid": 1336757, "tid": 1336757,
    "ts": 1514658039252.571, "dur": 20.384,
    "args": {
      "External id": 2489776,"Record function id": 0, "Concrete Inputs": ["", "inf"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11335
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::is_nonzero", "pid": 1336757, "tid": 1336757,
    "ts": 1514658039276.255, "dur": 35.825,
    "args": {
      "External id": 2489777,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11336
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::item", "pid": 1336757, "tid": 1336757,
    "ts": 1514658039279.009, "dur": 32.942,
    "args": {
      "External id": 2489778,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11337
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_local_scalar_dense", "pid": 1336757, "tid": 1336757,
    "ts": 1514658039279.835, "dur": 31.731,
    "args": {
      "External id": 2489779,"Sequence number": 24740658, "Fwd thread id": 0, "Record function id": 0, "Concrete Inputs": [""], "Input type": ["bool"], "Input Strides": [[]], "Input Dims": [[]], "Ev Idx": 11338
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "Optimizer.step#OptimizersContainer.step", "pid": 1336757, "tid": 1336757,
    "ts": 1514658039351.323, "dur": 5613.791,
    "args": {
      "External id": 2489780,"Record function id": 0, "Ev Idx": 11339
    }
  },
  {
    "ph": "X", "cat": "user_annotation", "name": "Optimizer.step#AdamW.step", "pid": 1336757, "tid": 1336757,
    "ts": 1514658039385.735, "dur": 5560.012,
    "args": {
      "External id": 2489781,"Record function id": 0, "Ev Idx": 11340
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_foreach_add_", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040621.043, "dur": 275.484,
    "args": {
      "External id": 2489782,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["TensorList", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11341
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040640.240, "dur": 1.230,
    "args": {
      "External id": 2489783,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11342
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040642.762, "dur": 0.372,
    "args": {
      "External id": 2489784,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11343
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040643.731, "dur": 0.069,
    "args": {
      "External id": 2489785,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11344
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040644.325, "dur": 0.321,
    "args": {
      "External id": 2489786,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11345
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040645.238, "dur": 0.194,
    "args": {
      "External id": 2489787,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11346
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040646.015, "dur": 0.101,
    "args": {
      "External id": 2489788,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11347
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040646.643, "dur": 0.063,
    "args": {
      "External id": 2489789,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11348
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040647.089, "dur": 0.052,
    "args": {
      "External id": 2489790,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11349
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040647.634, "dur": 0.068,
    "args": {
      "External id": 2489791,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11350
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040648.082, "dur": 0.068,
    "args": {
      "External id": 2489792,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11351
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040648.593, "dur": 0.065,
    "args": {
      "External id": 2489793,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11352
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040649.123, "dur": 0.061,
    "args": {
      "External id": 2489794,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11353
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040649.807, "dur": 0.074,
    "args": {
      "External id": 2489795,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11354
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040650.216, "dur": 0.066,
    "args": {
      "External id": 2489796,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11355
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040650.817, "dur": 0.068,
    "args": {
      "External id": 2489797,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11356
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040651.213, "dur": 0.064,
    "args": {
      "External id": 2489798,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11357
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040651.753, "dur": 0.066,
    "args": {
      "External id": 2489799,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11358
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040652.159, "dur": 0.198,
    "args": {
      "External id": 2489800,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11359
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040652.664, "dur": 0.200,
    "args": {
      "External id": 2489801,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11360
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040653.447, "dur": 0.171,
    "args": {
      "External id": 2489802,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11361
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040654.030, "dur": 0.207,
    "args": {
      "External id": 2489803,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11362
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040654.693, "dur": 0.218,
    "args": {
      "External id": 2489804,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11363
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040655.277, "dur": 0.066,
    "args": {
      "External id": 2489805,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11364
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040655.863, "dur": 0.054,
    "args": {
      "External id": 2489806,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11365
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040656.476, "dur": 0.066,
    "args": {
      "External id": 2489807,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11366
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040656.919, "dur": 0.065,
    "args": {
      "External id": 2489808,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11367
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040657.412, "dur": 0.065,
    "args": {
      "External id": 2489809,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11368
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040657.828, "dur": 0.059,
    "args": {
      "External id": 2489810,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11369
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040658.277, "dur": 0.053,
    "args": {
      "External id": 2489811,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11370
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040658.671, "dur": 0.064,
    "args": {
      "External id": 2489812,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11371
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040659.222, "dur": 0.065,
    "args": {
      "External id": 2489813,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11372
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040659.679, "dur": 0.067,
    "args": {
      "External id": 2489814,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11373
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040660.203, "dur": 0.063,
    "args": {
      "External id": 2489815,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11374
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040660.734, "dur": 0.063,
    "args": {
      "External id": 2489816,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11375
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040661.175, "dur": 0.061,
    "args": {
      "External id": 2489817,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11376
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040661.710, "dur": 0.066,
    "args": {
      "External id": 2489818,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11377
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040662.135, "dur": 0.064,
    "args": {
      "External id": 2489819,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11378
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040662.544, "dur": 0.055,
    "args": {
      "External id": 2489820,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11379
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040663.075, "dur": 0.064,
    "args": {
      "External id": 2489821,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11380
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040663.520, "dur": 0.063,
    "args": {
      "External id": 2489822,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11381
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040664.058, "dur": 0.067,
    "args": {
      "External id": 2489823,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11382
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040664.487, "dur": 0.065,
    "args": {
      "External id": 2489824,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11383
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040664.973, "dur": 0.056,
    "args": {
      "External id": 2489825,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11384
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040665.439, "dur": 0.084,
    "args": {
      "External id": 2489826,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11385
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040665.882, "dur": 0.217,
    "args": {
      "External id": 2489827,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11386
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040666.389, "dur": 0.114,
    "args": {
      "External id": 2489828,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11387
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040667.045, "dur": 0.059,
    "args": {
      "External id": 2489829,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11388
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040667.446, "dur": 0.077,
    "args": {
      "External id": 2489830,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11389
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040667.951, "dur": 0.052,
    "args": {
      "External id": 2489831,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11390
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040668.355, "dur": 0.078,
    "args": {
      "External id": 2489832,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11391
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040668.748, "dur": 0.088,
    "args": {
      "External id": 2489833,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11392
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040669.356, "dur": 0.089,
    "args": {
      "External id": 2489834,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11393
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040669.833, "dur": 0.089,
    "args": {
      "External id": 2489835,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11394
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040670.256, "dur": 0.068,
    "args": {
      "External id": 2489836,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11395
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040670.744, "dur": 0.065,
    "args": {
      "External id": 2489837,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11396
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040671.150, "dur": 0.065,
    "args": {
      "External id": 2489838,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11397
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040671.597, "dur": 0.067,
    "args": {
      "External id": 2489839,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11398
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040672.102, "dur": 0.066,
    "args": {
      "External id": 2489840,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11399
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040672.549, "dur": 0.066,
    "args": {
      "External id": 2489841,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11400
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040672.946, "dur": 0.067,
    "args": {
      "External id": 2489842,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11401
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040673.482, "dur": 0.047,
    "args": {
      "External id": 2489843,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11402
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040673.951, "dur": 0.056,
    "args": {
      "External id": 2489844,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11403
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040674.453, "dur": 0.064,
    "args": {
      "External id": 2489845,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11404
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040674.895, "dur": 0.060,
    "args": {
      "External id": 2489846,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11405
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040675.486, "dur": 0.063,
    "args": {
      "External id": 2489847,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11406
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040675.929, "dur": 0.060,
    "args": {
      "External id": 2489848,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11407
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040676.397, "dur": 0.055,
    "args": {
      "External id": 2489849,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11408
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040676.911, "dur": 0.067,
    "args": {
      "External id": 2489850,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11409
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040677.469, "dur": 0.055,
    "args": {
      "External id": 2489851,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11410
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040677.973, "dur": 0.062,
    "args": {
      "External id": 2489852,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11411
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040678.370, "dur": 0.065,
    "args": {
      "External id": 2489853,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11412
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040678.717, "dur": 0.065,
    "args": {
      "External id": 2489854,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11413
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040679.419, "dur": 0.066,
    "args": {
      "External id": 2489855,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11414
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040679.988, "dur": 0.064,
    "args": {
      "External id": 2489856,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11415
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040707.398, "dur": 0.060,
    "args": {
      "External id": 2489857,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11416
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040708.324, "dur": 0.046,
    "args": {
      "External id": 2489858,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11417
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040708.732, "dur": 0.061,
    "args": {
      "External id": 2489859,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11418
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040709.080, "dur": 0.067,
    "args": {
      "External id": 2489860,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11419
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040709.641, "dur": 0.059,
    "args": {
      "External id": 2489861,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11420
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040710.065, "dur": 0.052,
    "args": {
      "External id": 2489862,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11421
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040710.774, "dur": 0.060,
    "args": {
      "External id": 2489863,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11422
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040711.334, "dur": 0.065,
    "args": {
      "External id": 2489864,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11423
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040712.012, "dur": 0.062,
    "args": {
      "External id": 2489865,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11424
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040712.652, "dur": 0.067,
    "args": {
      "External id": 2489866,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11425
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040713.144, "dur": 0.062,
    "args": {
      "External id": 2489867,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11426
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040713.825, "dur": 0.064,
    "args": {
      "External id": 2489868,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11427
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040714.140, "dur": 0.061,
    "args": {
      "External id": 2489869,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11428
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040714.578, "dur": 0.066,
    "args": {
      "External id": 2489870,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11429
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040715.126, "dur": 0.068,
    "args": {
      "External id": 2489871,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11430
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040715.608, "dur": 0.063,
    "args": {
      "External id": 2489872,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11431
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040716.033, "dur": 0.055,
    "args": {
      "External id": 2489873,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11432
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040716.397, "dur": 0.055,
    "args": {
      "External id": 2489874,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11433
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040716.785, "dur": 0.063,
    "args": {
      "External id": 2489875,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11434
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040717.259, "dur": 0.054,
    "args": {
      "External id": 2489876,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11435
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040717.791, "dur": 0.062,
    "args": {
      "External id": 2489877,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11436
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040718.226, "dur": 0.062,
    "args": {
      "External id": 2489878,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11437
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040718.661, "dur": 0.057,
    "args": {
      "External id": 2489879,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11438
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040719.120, "dur": 0.064,
    "args": {
      "External id": 2489880,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11439
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040719.575, "dur": 0.065,
    "args": {
      "External id": 2489881,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11440
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040720.113, "dur": 0.058,
    "args": {
      "External id": 2489882,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11441
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040720.519, "dur": 0.064,
    "args": {
      "External id": 2489883,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11442
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040720.967, "dur": 0.047,
    "args": {
      "External id": 2489884,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11443
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040721.486, "dur": 0.054,
    "args": {
      "External id": 2489885,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11444
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040721.968, "dur": 0.066,
    "args": {
      "External id": 2489886,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11445
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040722.572, "dur": 0.058,
    "args": {
      "External id": 2489887,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11446
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040722.966, "dur": 0.066,
    "args": {
      "External id": 2489888,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11447
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040723.333, "dur": 0.068,
    "args": {
      "External id": 2489889,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11448
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040723.772, "dur": 0.065,
    "args": {
      "External id": 2489890,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11449
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040724.170, "dur": 0.069,
    "args": {
      "External id": 2489891,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11450
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040724.666, "dur": 0.088,
    "args": {
      "External id": 2489892,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11451
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040725.132, "dur": 0.065,
    "args": {
      "External id": 2489893,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11452
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040725.526, "dur": 0.061,
    "args": {
      "External id": 2489894,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11453
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040725.895, "dur": 0.067,
    "args": {
      "External id": 2489895,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11454
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040726.232, "dur": 0.065,
    "args": {
      "External id": 2489896,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11455
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040726.600, "dur": 0.062,
    "args": {
      "External id": 2489897,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11456
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040727.032, "dur": 0.068,
    "args": {
      "External id": 2489898,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11457
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040727.464, "dur": 0.056,
    "args": {
      "External id": 2489899,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11458
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040727.879, "dur": 0.067,
    "args": {
      "External id": 2489900,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11459
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040728.313, "dur": 0.067,
    "args": {
      "External id": 2489901,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11460
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040728.703, "dur": 0.062,
    "args": {
      "External id": 2489902,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11461
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040729.273, "dur": 0.067,
    "args": {
      "External id": 2489903,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11462
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040729.605, "dur": 0.068,
    "args": {
      "External id": 2489904,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11463
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040730.019, "dur": 0.063,
    "args": {
      "External id": 2489905,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11464
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040730.443, "dur": 0.051,
    "args": {
      "External id": 2489906,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11465
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040730.774, "dur": 0.054,
    "args": {
      "External id": 2489907,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11466
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040731.391, "dur": 0.066,
    "args": {
      "External id": 2489908,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11467
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040731.899, "dur": 0.050,
    "args": {
      "External id": 2489909,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11468
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040737.652, "dur": 0.080,
    "args": {
      "External id": 2489910,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11469
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040738.777, "dur": 0.065,
    "args": {
      "External id": 2489911,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11470
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040739.270, "dur": 0.063,
    "args": {
      "External id": 2489912,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11471
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040739.843, "dur": 0.054,
    "args": {
      "External id": 2489913,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11472
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040740.401, "dur": 0.062,
    "args": {
      "External id": 2489914,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11473
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040740.774, "dur": 0.065,
    "args": {
      "External id": 2489915,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11474
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040741.286, "dur": 0.054,
    "args": {
      "External id": 2489916,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11475
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040741.680, "dur": 0.068,
    "args": {
      "External id": 2489917,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11476
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040742.151, "dur": 0.070,
    "args": {
      "External id": 2489918,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11477
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040742.754, "dur": 0.067,
    "args": {
      "External id": 2489919,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11478
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040743.119, "dur": 0.062,
    "args": {
      "External id": 2489920,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11479
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040743.597, "dur": 0.055,
    "args": {
      "External id": 2489921,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11480
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040744.051, "dur": 0.054,
    "args": {
      "External id": 2489922,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11481
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040744.604, "dur": 0.053,
    "args": {
      "External id": 2489923,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11482
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040745.039, "dur": 0.055,
    "args": {
      "External id": 2489924,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11483
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040745.641, "dur": 0.058,
    "args": {
      "External id": 2489925,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11484
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040746.101, "dur": 0.067,
    "args": {
      "External id": 2489926,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11485
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040746.605, "dur": 0.068,
    "args": {
      "External id": 2489927,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11486
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040747.113, "dur": 0.067,
    "args": {
      "External id": 2489928,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11487
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040747.647, "dur": 0.065,
    "args": {
      "External id": 2489929,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11488
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040748.077, "dur": 0.065,
    "args": {
      "External id": 2489930,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11489
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040748.571, "dur": 0.057,
    "args": {
      "External id": 2489931,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11490
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040748.990, "dur": 0.063,
    "args": {
      "External id": 2489932,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11491
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040749.424, "dur": 0.063,
    "args": {
      "External id": 2489933,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11492
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040750.005, "dur": 0.069,
    "args": {
      "External id": 2489934,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11493
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040750.516, "dur": 0.068,
    "args": {
      "External id": 2489935,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11494
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040750.917, "dur": 0.061,
    "args": {
      "External id": 2489936,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11495
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040751.430, "dur": 0.067,
    "args": {
      "External id": 2489937,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11496
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040751.843, "dur": 0.065,
    "args": {
      "External id": 2489938,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11497
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040752.231, "dur": 0.057,
    "args": {
      "External id": 2489939,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11498
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040752.763, "dur": 0.054,
    "args": {
      "External id": 2489940,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11499
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040753.165, "dur": 0.063,
    "args": {
      "External id": 2489941,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11500
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040753.584, "dur": 0.068,
    "args": {
      "External id": 2489942,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11501
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040754.095, "dur": 0.057,
    "args": {
      "External id": 2489943,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11502
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040754.498, "dur": 0.047,
    "args": {
      "External id": 2489944,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11503
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040755.012, "dur": 0.056,
    "args": {
      "External id": 2489945,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11504
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040755.464, "dur": 0.050,
    "args": {
      "External id": 2489946,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11505
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040755.846, "dur": 0.066,
    "args": {
      "External id": 2489947,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11506
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040756.362, "dur": 0.066,
    "args": {
      "External id": 2489948,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11507
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040756.824, "dur": 0.066,
    "args": {
      "External id": 2489949,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11508
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040757.345, "dur": 0.053,
    "args": {
      "External id": 2489950,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11509
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040757.731, "dur": 0.059,
    "args": {
      "External id": 2489951,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11510
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040758.170, "dur": 0.061,
    "args": {
      "External id": 2489952,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11511
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040758.558, "dur": 0.057,
    "args": {
      "External id": 2489953,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11512
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040758.998, "dur": 0.069,
    "args": {
      "External id": 2489954,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11513
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040759.386, "dur": 0.065,
    "args": {
      "External id": 2489955,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11514
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040759.771, "dur": 0.067,
    "args": {
      "External id": 2489956,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11515
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040760.185, "dur": 0.052,
    "args": {
      "External id": 2489957,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11516
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040760.790, "dur": 0.055,
    "args": {
      "External id": 2489958,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11517
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040761.175, "dur": 0.066,
    "args": {
      "External id": 2489959,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11518
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040761.625, "dur": 0.068,
    "args": {
      "External id": 2489960,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11519
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040762.037, "dur": 0.063,
    "args": {
      "External id": 2489961,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11520
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040762.427, "dur": 0.058,
    "args": {
      "External id": 2489962,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11521
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040762.741, "dur": 0.071,
    "args": {
      "External id": 2489963,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11522
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040763.181, "dur": 0.068,
    "args": {
      "External id": 2489964,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11523
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040763.554, "dur": 0.069,
    "args": {
      "External id": 2489965,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11524
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040763.996, "dur": 0.065,
    "args": {
      "External id": 2489966,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11525
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040764.630, "dur": 0.066,
    "args": {
      "External id": 2489967,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11526
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040765.181, "dur": 0.057,
    "args": {
      "External id": 2489968,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11527
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040765.943, "dur": 0.059,
    "args": {
      "External id": 2489969,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11528
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040766.324, "dur": 0.066,
    "args": {
      "External id": 2489970,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11529
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040766.769, "dur": 0.065,
    "args": {
      "External id": 2489971,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11530
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040767.247, "dur": 0.063,
    "args": {
      "External id": 2489972,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11531
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040767.734, "dur": 0.072,
    "args": {
      "External id": 2489973,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11532
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040768.112, "dur": 0.070,
    "args": {
      "External id": 2489974,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11533
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040768.570, "dur": 0.068,
    "args": {
      "External id": 2489975,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11534
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040769.049, "dur": 0.068,
    "args": {
      "External id": 2489976,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11535
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040769.495, "dur": 0.064,
    "args": {
      "External id": 2489977,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11536
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040769.842, "dur": 0.066,
    "args": {
      "External id": 2489978,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11537
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040770.312, "dur": 0.045,
    "args": {
      "External id": 2489979,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11538
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040770.835, "dur": 0.056,
    "args": {
      "External id": 2489980,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11539
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040771.318, "dur": 0.055,
    "args": {
      "External id": 2489981,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11540
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040771.730, "dur": 0.060,
    "args": {
      "External id": 2489982,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11541
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040772.077, "dur": 0.065,
    "args": {
      "External id": 2489983,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11542
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040772.584, "dur": 0.060,
    "args": {
      "External id": 2489984,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11543
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040773.087, "dur": 0.065,
    "args": {
      "External id": 2489985,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11544
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040773.454, "dur": 0.065,
    "args": {
      "External id": 2489986,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11545
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040773.773, "dur": 0.049,
    "args": {
      "External id": 2489987,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11546
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040774.577, "dur": 0.066,
    "args": {
      "External id": 2489988,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11547
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040774.997, "dur": 0.065,
    "args": {
      "External id": 2489989,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11548
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040775.654, "dur": 0.052,
    "args": {
      "External id": 2489990,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11549
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040775.964, "dur": 0.054,
    "args": {
      "External id": 2489991,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11550
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040776.684, "dur": 0.053,
    "args": {
      "External id": 2489992,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11551
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040776.988, "dur": 0.056,
    "args": {
      "External id": 2489993,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11552
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040777.684, "dur": 0.064,
    "args": {
      "External id": 2489994,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11553
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040778.016, "dur": 0.048,
    "args": {
      "External id": 2489995,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11554
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040778.600, "dur": 0.068,
    "args": {
      "External id": 2489996,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11555
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040778.917, "dur": 0.056,
    "args": {
      "External id": 2489997,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11556
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040779.538, "dur": 0.058,
    "args": {
      "External id": 2489998,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11557
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040779.916, "dur": 0.051,
    "args": {
      "External id": 2489999,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11558
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040780.387, "dur": 0.057,
    "args": {
      "External id": 2490000,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11559
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040780.698, "dur": 0.052,
    "args": {
      "External id": 2490001,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11560
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040781.232, "dur": 0.061,
    "args": {
      "External id": 2490002,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11561
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040781.550, "dur": 0.048,
    "args": {
      "External id": 2490003,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11562
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040782.029, "dur": 0.056,
    "args": {
      "External id": 2490004,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11563
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040782.342, "dur": 0.056,
    "args": {
      "External id": 2490005,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11564
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040782.867, "dur": 0.064,
    "args": {
      "External id": 2490006,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11565
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040783.181, "dur": 0.055,
    "args": {
      "External id": 2490007,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11566
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040783.875, "dur": 0.060,
    "args": {
      "External id": 2490008,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11567
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040784.508, "dur": 0.062,
    "args": {
      "External id": 2490009,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11568
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040785.091, "dur": 0.066,
    "args": {
      "External id": 2490010,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11569
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040785.430, "dur": 0.052,
    "args": {
      "External id": 2490011,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11570
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040786.106, "dur": 0.064,
    "args": {
      "External id": 2490012,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11571
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040786.421, "dur": 0.046,
    "args": {
      "External id": 2490013,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11572
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040787.005, "dur": 0.067,
    "args": {
      "External id": 2490014,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11573
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040787.332, "dur": 0.051,
    "args": {
      "External id": 2490015,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11574
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040787.964, "dur": 0.063,
    "args": {
      "External id": 2490016,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11575
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040788.339, "dur": 0.053,
    "args": {
      "External id": 2490017,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11576
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040788.901, "dur": 0.064,
    "args": {
      "External id": 2490018,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11577
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040789.220, "dur": 0.052,
    "args": {
      "External id": 2490019,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11578
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040789.670, "dur": 0.054,
    "args": {
      "External id": 2490020,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11579
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040789.981, "dur": 0.047,
    "args": {
      "External id": 2490021,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11580
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040790.608, "dur": 0.056,
    "args": {
      "External id": 2490022,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11581
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040790.917, "dur": 0.051,
    "args": {
      "External id": 2490023,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11582
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040791.585, "dur": 0.068,
    "args": {
      "External id": 2490024,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11583
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040791.909, "dur": 0.057,
    "args": {
      "External id": 2490025,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11584
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040792.559, "dur": 0.062,
    "args": {
      "External id": 2490026,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11585
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040792.877, "dur": 0.053,
    "args": {
      "External id": 2490027,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11586
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040793.556, "dur": 0.056,
    "args": {
      "External id": 2490028,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11587
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040793.870, "dur": 0.051,
    "args": {
      "External id": 2490029,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11588
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040794.506, "dur": 0.065,
    "args": {
      "External id": 2490030,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11589
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040794.830, "dur": 0.048,
    "args": {
      "External id": 2490031,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11590
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040795.451, "dur": 0.063,
    "args": {
      "External id": 2490032,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11591
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040795.808, "dur": 0.053,
    "args": {
      "External id": 2490033,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11592
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040796.423, "dur": 0.062,
    "args": {
      "External id": 2490034,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11593
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040796.810, "dur": 0.052,
    "args": {
      "External id": 2490035,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11594
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040797.447, "dur": 0.055,
    "args": {
      "External id": 2490036,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11595
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040797.756, "dur": 0.053,
    "args": {
      "External id": 2490037,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11596
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040798.260, "dur": 0.063,
    "args": {
      "External id": 2490038,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11597
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040798.578, "dur": 0.054,
    "args": {
      "External id": 2490039,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11598
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040799.232, "dur": 0.064,
    "args": {
      "External id": 2490040,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11599
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040799.546, "dur": 0.054,
    "args": {
      "External id": 2490041,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11600
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040800.049, "dur": 0.054,
    "args": {
      "External id": 2490042,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11601
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040800.363, "dur": 0.046,
    "args": {
      "External id": 2490043,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11602
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040800.921, "dur": 0.062,
    "args": {
      "External id": 2490044,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11603
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040801.288, "dur": 0.047,
    "args": {
      "External id": 2490045,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11604
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040801.913, "dur": 0.056,
    "args": {
      "External id": 2490046,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11605
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040802.224, "dur": 0.047,
    "args": {
      "External id": 2490047,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11606
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040802.838, "dur": 0.062,
    "args": {
      "External id": 2490048,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11607
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040803.159, "dur": 0.049,
    "args": {
      "External id": 2490049,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11608
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040803.821, "dur": 0.066,
    "args": {
      "External id": 2490050,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11609
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040804.138, "dur": 0.054,
    "args": {
      "External id": 2490051,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11610
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040804.703, "dur": 0.056,
    "args": {
      "External id": 2490052,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11611
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040805.016, "dur": 0.050,
    "args": {
      "External id": 2490053,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11612
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040805.489, "dur": 0.069,
    "args": {
      "External id": 2490054,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11613
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040805.826, "dur": 0.048,
    "args": {
      "External id": 2490055,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11614
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040806.376, "dur": 0.054,
    "args": {
      "External id": 2490056,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11615
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040806.681, "dur": 0.053,
    "args": {
      "External id": 2490057,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11616
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040807.168, "dur": 0.067,
    "args": {
      "External id": 2490058,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11617
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040807.492, "dur": 0.053,
    "args": {
      "External id": 2490059,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11618
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040808.116, "dur": 0.066,
    "args": {
      "External id": 2490060,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11619
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040808.438, "dur": 0.049,
    "args": {
      "External id": 2490061,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11620
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040809.105, "dur": 0.066,
    "args": {
      "External id": 2490062,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11621
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040809.425, "dur": 0.047,
    "args": {
      "External id": 2490063,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11622
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040809.870, "dur": 0.063,
    "args": {
      "External id": 2490064,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11623
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040810.192, "dur": 0.046,
    "args": {
      "External id": 2490065,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11624
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040810.698, "dur": 0.062,
    "args": {
      "External id": 2490066,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11625
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040811.035, "dur": 0.052,
    "args": {
      "External id": 2490067,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11626
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040811.534, "dur": 0.065,
    "args": {
      "External id": 2490068,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11627
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040811.853, "dur": 0.053,
    "args": {
      "External id": 2490069,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11628
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040812.415, "dur": 0.066,
    "args": {
      "External id": 2490070,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11629
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040812.734, "dur": 0.053,
    "args": {
      "External id": 2490071,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11630
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040813.395, "dur": 0.063,
    "args": {
      "External id": 2490072,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11631
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::result_type", "pid": 1336757, "tid": 1336757,
    "ts": 1514658040813.715, "dur": 0.076,
    "args": {
      "External id": 2490073,"Record function id": 0, "Concrete Inputs": ["", "1"], "Input type": ["float", "Scalar"], "Input Strides": [[], []], "Input Dims": [[], []], "Ev Idx": 11632
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_fused_adamw_", "pid": 1336757, "tid": 1336757,
    "ts": 1514658041409.695, "dur": 3441.964,
    "args": {
      "External id": 2490074,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "0.0001072708166170532", "0.90000000000000002", "0.94999999999999996", "0.10000000000000001", "1.0000000000000001e-15", "False", "False", "", ""], "Input type": ["TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 11633
    }
  },
  {
    "ph": "X", "cat": "cpu_op", "name": "aten::_fused_adamw_", "pid": 1336757, "tid": 1336757,
    "ts": 1514658044370.148, "dur": 328.651,
    "args": {
      "External id": 2490075,"Record function id": 0, "Concrete Inputs": ["", "", "", "", "", "", "0.0001072708166170532", "0.90000000000000002", "0.94999999999999996", "0.10000000000000001", "1.0000000000000001e-15", "False", "False", "", ""], "Input type": ["TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "TensorList", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "Scalar", "", ""], "Input Strides": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Input Dims": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "Ev Idx": 11634
    }
  },
  {
    "name": "process_name", "ph": "M", "ts": 1514655936773.108, "pid": 1336757, "tid": 0,
    "args": {
      "name": "python3.12"
    }
  },
  {
    "name": "process_labels", "ph": "M", "ts": 1514655936773.108, "pid": 1336757, "tid": 0,
    "args": {
      "labels": "CPU"
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1514655936773.108, "pid": 1336757, "tid": 0,
    "args": {
      "sort_index": 1336757
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1514655936773.108, "pid": 1336757, "tid": 1381170,
    "args": {
      "name": "thread 1381170 (pt_autograd_4)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1514655936773.108, "pid": 1336757, "tid": 1381170,
    "args": {
      "sort_index": 1381170
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1514655936773.108, "pid": 1336757, "tid": 1336757,
    "args": {
      "name": "thread 1336757 (python3.12)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1514655936773.108, "pid": 1336757, "tid": 1336757,
    "args": {
      "sort_index": 1336757
    }
  },
  {
    "name": "thread_name", "ph": "M", "ts": 1514655936773.108, "pid": 1336757, "tid": 1381170,
    "args": {
      "name": "thread 1381170 (python3.12)"
    }
  },
  {
    "name": "thread_sort_index", "ph": "M", "ts": 1514655936773.108, "pid": 1336757, "tid": 1381170,
    "args": {
      "sort_index": 1381170
    }
  },
  {
    "ph": "X", "cat": "Trace", "ts": 1514655936699.080, "dur": 2109980.672,
    "pid": "Spans", "tid": "PyTorch Profiler",
    "name": "PyTorch Profiler (0)",
    "args": {
      "Op count": 0
    }
  },
  {
    "name": "process_sort_index", "ph": "M", "ts": 1514655936699.080,
    "pid": "Spans", "tid": 0,
    "args": {
      "sort_index": 536870912
    }
  },
  {
    "name": "Iteration Start: PyTorch Profiler", "ph": "i", "s": "g",
    "pid": "Traces", "tid": "Trace PyTorch Profiler", "ts": 1514655936699.080
  },
  {
    "name": "Record Window End", "ph": "i", "s": "g",
    "pid": "", "tid": "", "ts": 1514658111339.353
  }
  ],
  "traceName": "exp/mtp.1B.batch16.seqlen4096.context4096.warmup2000.update1.steps200000.lr2e-4.cosine/profile_trace/iteration_102912/rank4_trace.json",
  "displayTimeUnit": "ms",
  "baseTimeNanoseconds": 1751410836000000000
}